{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'%.2f'"
      ]
     },
     "execution_count": 216,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from cot import Collection\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import krippendorff\n",
    "import random\n",
    "import scipy\n",
    "from scipy import stats\n",
    "pd.set_option('display.max_rows', 200)\n",
    "pd.options.display.float_format = \"{:.2f}\".format\n",
    "%precision 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [],
   "source": [
    "coll = Collection.from_json(\"/home/kon/work/ThoughtSource/libs/cot/cot/datasets/thoughtsource/thoughtsource_33_paper.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from cot.evaluate import compare_evaluation_difference\n",
    "# compare_evaluation_difference(coll)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = coll.collection_to_dataframe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>split</th>\n",
       "      <th>id</th>\n",
       "      <th>type</th>\n",
       "      <th>answer_label</th>\n",
       "      <th>prompt</th>\n",
       "      <th>instruction</th>\n",
       "      <th>cot_trigger</th>\n",
       "      <th>answer_id</th>\n",
       "      <th>answer_from_choices</th>\n",
       "      <th>correct_answer</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>commonsense_qa</td>\n",
       "      <td>validation</td>\n",
       "      <td>9aff72f0c480c2b4edde45bd2e7e4870</td>\n",
       "      <td>multiplechoice</td>\n",
       "      <td>E</td>\n",
       "      <td>None_None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>20897a78-9ce5-45f4-8f5f-6e50b5c8fd65</td>\n",
       "      <td>E</td>\n",
       "      <td>True</td>\n",
       "      <td>command-xlarge-nightly</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          dataset       split                                id  \\\n",
       "0  commonsense_qa  validation  9aff72f0c480c2b4edde45bd2e7e4870   \n",
       "\n",
       "             type answer_label     prompt instruction cot_trigger  \\\n",
       "0  multiplechoice            E  None_None        None        None   \n",
       "\n",
       "                              answer_id answer_from_choices correct_answer  \\\n",
       "0  20897a78-9ce5-45f4-8f5f-6e50b5c8fd65                   E           True   \n",
       "\n",
       "                    model  \n",
       "0  command-xlarge-nightly  "
      ]
     },
     "execution_count": 225,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Series([], Name: answer_from_choices, dtype: int64)"
      ]
     },
     "execution_count": 226,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['correct_answer'].isna()][\"answer_from_choices\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Series([], Name: correct_answer, dtype: int64)"
      ]
     },
     "execution_count": 227,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['answer_from_choices'].isna()][\"correct_answer\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check if those items are from StrategyQA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>split</th>\n",
       "      <th>id</th>\n",
       "      <th>type</th>\n",
       "      <th>answer_label</th>\n",
       "      <th>prompt</th>\n",
       "      <th>instruction</th>\n",
       "      <th>cot_trigger</th>\n",
       "      <th>answer_id</th>\n",
       "      <th>answer_from_choices</th>\n",
       "      <th>correct_answer</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [dataset, split, id, type, answer_label, prompt, instruction, cot_trigger, answer_id, answer_from_choices, correct_answer, model]\n",
       "Index: []"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[(df['answer_from_choices'].isna()) & (df['correct_answer'] == False)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "model\n",
       "command-xlarge-nightly     18\n",
       "flan-T5-xxl                28\n",
       "gpt-3.5-turbo             105\n",
       "gpt-4                      63\n",
       "text-davinci-002           55\n",
       "text-davinci-003           78\n",
       "Name: correct_answer, dtype: int64"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ammount of NaN anwers per model. These are inconclusive answers given, and the evaluation code does not take them into account.\n",
    "df.groupby([\"model\"]).correct_answer.apply(lambda x: x.isnull().sum())"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_confidence_intervals(dataframe, index_col, columns_col, values_col, confidence_level=0.95):\n",
    "    df = dataframe.copy()\n",
    "    df = df.dropna(subset=['correct_answer'])\n",
    "    df[\"correct_answer\"] = df[\"correct_answer\"].fillna(np.nan)\n",
    "    from scipy.stats import t\n",
    "    # Calculate mean accuracy scores\n",
    "    acc = df.groupby([columns_col, index_col]).mean().pivot_table(index=index_col, columns=columns_col, values=values_col)\n",
    "\n",
    "    # Calculate standard error for each group\n",
    "    standard_errors = df.groupby([columns_col, index_col]).sem().pivot_table(index=index_col, columns=columns_col, values=values_col)\n",
    "\n",
    "    # Calculate standard deviations for each group\n",
    "    standard_deviations = df.groupby([columns_col, index_col]).std().pivot_table(index=index_col, columns=columns_col, values=values_col)\n",
    "\n",
    "    # Calculate degrees of freedom for each group\n",
    "    degrees_freedom = df.groupby([columns_col, index_col]).count().pivot_table(index=index_col, columns=columns_col, values=values_col) - 1\n",
    "\n",
    "    # Calculate the t-score for the chosen confidence level\n",
    "    t_scores = t.ppf((1 + confidence_level) / 2, degrees_freedom)\n",
    "\n",
    "    # Calculate the margin of error for each group\n",
    "    margin_of_error = t_scores * standard_errors\n",
    "\n",
    "    # Calculate the confidence intervals\n",
    "    lower_bound = acc - margin_of_error\n",
    "    upper_bound = acc + margin_of_error\n",
    "\n",
    "    return acc, standard_deviations, lower_bound, upper_bound"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1153/3665219877.py:7: FutureWarning: The default value of numeric_only in DataFrameGroupBy.mean is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  acc = df.groupby([columns_col, index_col]).mean().pivot_table(index=index_col, columns=columns_col, values=values_col)\n",
      "/tmp/ipykernel_1153/3665219877.py:10: FutureWarning: The default value of numeric_only in DataFrameGroupBy.sem is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  standard_errors = df.groupby([columns_col, index_col]).sem().pivot_table(index=index_col, columns=columns_col, values=values_col)\n",
      "/tmp/ipykernel_1153/3665219877.py:13: FutureWarning: The default value of numeric_only in DataFrameGroupBy.std is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  standard_deviations = df.groupby([columns_col, index_col]).std().pivot_table(index=index_col, columns=columns_col, values=values_col)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>prompt</th>\n",
       "      <th>None_None</th>\n",
       "      <th>None_kojima-01</th>\n",
       "      <th>None_zhou-01</th>\n",
       "      <th>qa-10_None</th>\n",
       "      <th>qa-12_None</th>\n",
       "      <th>qa-13_None</th>\n",
       "      <th>qa-16_None</th>\n",
       "      <th>qa-17_None</th>\n",
       "      <th>refl-01_None</th>\n",
       "      <th>zhou-01-ins_None</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>command-xlarge-nightly</th>\n",
       "      <td>0.47</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.55</td>\n",
       "      <td>0.53</td>\n",
       "      <td>0.53</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.55</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>flan-T5-xxl</th>\n",
       "      <td>0.62</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.59</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gpt-3.5-turbo</th>\n",
       "      <td>0.75</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.74</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.71</td>\n",
       "      <td>0.72</td>\n",
       "      <td>0.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gpt-4</th>\n",
       "      <td>0.81</td>\n",
       "      <td>0.86</td>\n",
       "      <td>0.89</td>\n",
       "      <td>0.84</td>\n",
       "      <td>0.86</td>\n",
       "      <td>0.84</td>\n",
       "      <td>0.84</td>\n",
       "      <td>0.81</td>\n",
       "      <td>0.84</td>\n",
       "      <td>0.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>text-davinci-002</th>\n",
       "      <td>0.59</td>\n",
       "      <td>0.56</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.55</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.53</td>\n",
       "      <td>0.57</td>\n",
       "      <td>0.54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>text-davinci-003</th>\n",
       "      <td>0.61</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.65</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "prompt                  None_None  None_kojima-01  None_zhou-01  qa-10_None  \\\n",
       "model                                                                         \n",
       "command-xlarge-nightly       0.47            0.45          0.55        0.53   \n",
       "flan-T5-xxl                  0.62            0.62          0.58        0.62   \n",
       "gpt-3.5-turbo                0.75            0.76          0.75        0.73   \n",
       "gpt-4                        0.81            0.86          0.89        0.84   \n",
       "text-davinci-002             0.59            0.56          0.65        0.55   \n",
       "text-davinci-003             0.61            0.61          0.66        0.63   \n",
       "\n",
       "prompt                  qa-12_None  qa-13_None  qa-16_None  qa-17_None  \\\n",
       "model                                                                    \n",
       "command-xlarge-nightly        0.53        0.58        0.55        0.52   \n",
       "flan-T5-xxl                   0.65        0.62        0.59        0.61   \n",
       "gpt-3.5-turbo                 0.74        0.73        0.73        0.71   \n",
       "gpt-4                         0.86        0.84        0.84        0.81   \n",
       "text-davinci-002              0.61        0.61        0.65        0.53   \n",
       "text-davinci-003              0.65        0.63        0.60        0.62   \n",
       "\n",
       "prompt                  refl-01_None  zhou-01-ins_None  \n",
       "model                                                   \n",
       "command-xlarge-nightly          0.51              0.56  \n",
       "flan-T5-xxl                     0.58              0.59  \n",
       "gpt-3.5-turbo                   0.72              0.75  \n",
       "gpt-4                           0.84              0.85  \n",
       "text-davinci-002                0.57              0.54  \n",
       "text-davinci-003                0.64              0.65  "
      ]
     },
     "execution_count": 270,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc_model_prompt_mean, acc_model_prompt_std, acc_model_prompt_ci_lower, acc_model_prompt_ci_upper = calculate_confidence_intervals(df, \"model\", \"prompt\", \"correct_answer\", confidence_level=0.95)\n",
    "acc_model_prompt_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1153/3665219877.py:7: FutureWarning: The default value of numeric_only in DataFrameGroupBy.mean is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  acc = df.groupby([columns_col, index_col]).mean().pivot_table(index=index_col, columns=columns_col, values=values_col)\n",
      "/tmp/ipykernel_1153/3665219877.py:10: FutureWarning: The default value of numeric_only in DataFrameGroupBy.sem is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  standard_errors = df.groupby([columns_col, index_col]).sem().pivot_table(index=index_col, columns=columns_col, values=values_col)\n",
      "/tmp/ipykernel_1153/3665219877.py:13: FutureWarning: The default value of numeric_only in DataFrameGroupBy.std is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  standard_deviations = df.groupby([columns_col, index_col]).std().pivot_table(index=index_col, columns=columns_col, values=values_col)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>model</th>\n",
       "      <th>command-xlarge-nightly</th>\n",
       "      <th>flan-T5-xxl</th>\n",
       "      <th>gpt-3.5-turbo</th>\n",
       "      <th>gpt-4</th>\n",
       "      <th>text-davinci-002</th>\n",
       "      <th>text-davinci-003</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>commonsense_qa</th>\n",
       "      <td>0.66</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>med_qa</th>\n",
       "      <td>0.27</td>\n",
       "      <td>0.22</td>\n",
       "      <td>0.53</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.28</td>\n",
       "      <td>0.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>medmc_qa</th>\n",
       "      <td>0.31</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.41</td>\n",
       "      <td>0.41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>open_book_qa</th>\n",
       "      <td>0.58</td>\n",
       "      <td>0.78</td>\n",
       "      <td>0.83</td>\n",
       "      <td>0.93</td>\n",
       "      <td>0.59</td>\n",
       "      <td>0.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>strategy_qa</th>\n",
       "      <td>0.57</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>worldtree</th>\n",
       "      <td>0.75</td>\n",
       "      <td>0.83</td>\n",
       "      <td>0.92</td>\n",
       "      <td>0.98</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.88</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "model           command-xlarge-nightly  flan-T5-xxl  gpt-3.5-turbo  gpt-4  \\\n",
       "dataset                                                                     \n",
       "commonsense_qa                    0.66         0.85           0.76   0.85   \n",
       "med_qa                            0.27         0.22           0.53   0.65   \n",
       "medmc_qa                          0.31         0.35           0.63   0.80   \n",
       "open_book_qa                      0.58         0.78           0.83   0.93   \n",
       "strategy_qa                       0.57         0.62           0.73   0.85   \n",
       "worldtree                         0.75         0.83           0.92   0.98   \n",
       "\n",
       "model           text-davinci-002  text-davinci-003  \n",
       "dataset                                             \n",
       "commonsense_qa              0.75              0.75  \n",
       "med_qa                      0.28              0.35  \n",
       "medmc_qa                    0.41              0.41  \n",
       "open_book_qa                0.59              0.75  \n",
       "strategy_qa                 0.63              0.63  \n",
       "worldtree                   0.88              0.88  "
      ]
     },
     "execution_count": 271,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc_dataset_model_mean, acc_dataset_model_std, acc_dataset_model_ci_lower, acc_dataset_model_ci_upper = calculate_confidence_intervals(df, \"dataset\", \"model\", \"correct_answer\", confidence_level=0.95)\n",
    "acc_dataset_model_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1153/3665219877.py:7: FutureWarning: The default value of numeric_only in DataFrameGroupBy.mean is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  acc = df.groupby([columns_col, index_col]).mean().pivot_table(index=index_col, columns=columns_col, values=values_col)\n",
      "/tmp/ipykernel_1153/3665219877.py:10: FutureWarning: The default value of numeric_only in DataFrameGroupBy.sem is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  standard_errors = df.groupby([columns_col, index_col]).sem().pivot_table(index=index_col, columns=columns_col, values=values_col)\n",
      "/tmp/ipykernel_1153/3665219877.py:13: FutureWarning: The default value of numeric_only in DataFrameGroupBy.std is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  standard_deviations = df.groupby([columns_col, index_col]).std().pivot_table(index=index_col, columns=columns_col, values=values_col)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>prompt</th>\n",
       "      <th>None_None</th>\n",
       "      <th>None_kojima-01</th>\n",
       "      <th>None_zhou-01</th>\n",
       "      <th>qa-10_None</th>\n",
       "      <th>qa-12_None</th>\n",
       "      <th>qa-13_None</th>\n",
       "      <th>qa-16_None</th>\n",
       "      <th>qa-17_None</th>\n",
       "      <th>refl-01_None</th>\n",
       "      <th>zhou-01-ins_None</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>command-xlarge-nightly</th>\n",
       "      <td>0.47</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.55</td>\n",
       "      <td>0.53</td>\n",
       "      <td>0.53</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.55</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>flan-T5-xxl</th>\n",
       "      <td>0.62</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.59</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gpt-3.5-turbo</th>\n",
       "      <td>0.75</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.74</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.71</td>\n",
       "      <td>0.72</td>\n",
       "      <td>0.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gpt-4</th>\n",
       "      <td>0.81</td>\n",
       "      <td>0.86</td>\n",
       "      <td>0.89</td>\n",
       "      <td>0.84</td>\n",
       "      <td>0.86</td>\n",
       "      <td>0.84</td>\n",
       "      <td>0.84</td>\n",
       "      <td>0.81</td>\n",
       "      <td>0.84</td>\n",
       "      <td>0.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>text-davinci-002</th>\n",
       "      <td>0.59</td>\n",
       "      <td>0.56</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.55</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.53</td>\n",
       "      <td>0.57</td>\n",
       "      <td>0.54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>text-davinci-003</th>\n",
       "      <td>0.61</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.65</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "prompt                  None_None  None_kojima-01  None_zhou-01  qa-10_None  \\\n",
       "model                                                                         \n",
       "command-xlarge-nightly       0.47            0.45          0.55        0.53   \n",
       "flan-T5-xxl                  0.62            0.62          0.58        0.62   \n",
       "gpt-3.5-turbo                0.75            0.76          0.75        0.73   \n",
       "gpt-4                        0.81            0.86          0.89        0.84   \n",
       "text-davinci-002             0.59            0.56          0.65        0.55   \n",
       "text-davinci-003             0.61            0.61          0.66        0.63   \n",
       "\n",
       "prompt                  qa-12_None  qa-13_None  qa-16_None  qa-17_None  \\\n",
       "model                                                                    \n",
       "command-xlarge-nightly        0.53        0.58        0.55        0.52   \n",
       "flan-T5-xxl                   0.65        0.62        0.59        0.61   \n",
       "gpt-3.5-turbo                 0.74        0.73        0.73        0.71   \n",
       "gpt-4                         0.86        0.84        0.84        0.81   \n",
       "text-davinci-002              0.61        0.61        0.65        0.53   \n",
       "text-davinci-003              0.65        0.63        0.60        0.62   \n",
       "\n",
       "prompt                  refl-01_None  zhou-01-ins_None  \n",
       "model                                                   \n",
       "command-xlarge-nightly          0.51              0.56  \n",
       "flan-T5-xxl                     0.58              0.59  \n",
       "gpt-3.5-turbo                   0.72              0.75  \n",
       "gpt-4                           0.84              0.85  \n",
       "text-davinci-002                0.57              0.54  \n",
       "text-davinci-003                0.64              0.65  "
      ]
     },
     "execution_count": 272,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc_model_prompt_mean, acc_model_prompt_std, acc_model_prompt_ci_lower, acc_model_prompt_ci_upper = calculate_confidence_intervals(df, \"model\", \"prompt\", \"correct_answer\", confidence_level=0.95)\n",
    "acc_model_prompt_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1153/3665219877.py:7: FutureWarning: The default value of numeric_only in DataFrameGroupBy.mean is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  acc = df.groupby([columns_col, index_col]).mean().pivot_table(index=index_col, columns=columns_col, values=values_col)\n",
      "/tmp/ipykernel_1153/3665219877.py:10: FutureWarning: The default value of numeric_only in DataFrameGroupBy.sem is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  standard_errors = df.groupby([columns_col, index_col]).sem().pivot_table(index=index_col, columns=columns_col, values=values_col)\n",
      "/tmp/ipykernel_1153/3665219877.py:13: FutureWarning: The default value of numeric_only in DataFrameGroupBy.std is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  standard_deviations = df.groupby([columns_col, index_col]).std().pivot_table(index=index_col, columns=columns_col, values=values_col)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>prompt</th>\n",
       "      <th>None_None</th>\n",
       "      <th>None_kojima-01</th>\n",
       "      <th>None_zhou-01</th>\n",
       "      <th>qa-10_None</th>\n",
       "      <th>qa-12_None</th>\n",
       "      <th>qa-13_None</th>\n",
       "      <th>qa-16_None</th>\n",
       "      <th>qa-17_None</th>\n",
       "      <th>refl-01_None</th>\n",
       "      <th>zhou-01-ins_None</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>commonsense_qa</th>\n",
       "      <td>0.74</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.78</td>\n",
       "      <td>0.78</td>\n",
       "      <td>0.78</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.71</td>\n",
       "      <td>0.79</td>\n",
       "      <td>0.79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>med_qa</th>\n",
       "      <td>0.38</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.41</td>\n",
       "      <td>0.37</td>\n",
       "      <td>0.37</td>\n",
       "      <td>0.37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>medmc_qa</th>\n",
       "      <td>0.46</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.54</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.49</td>\n",
       "      <td>0.53</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>open_book_qa</th>\n",
       "      <td>0.74</td>\n",
       "      <td>0.71</td>\n",
       "      <td>0.81</td>\n",
       "      <td>0.74</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.71</td>\n",
       "      <td>0.74</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>strategy_qa</th>\n",
       "      <td>0.64</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.72</td>\n",
       "      <td>0.68</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>worldtree</th>\n",
       "      <td>0.88</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.87</td>\n",
       "      <td>0.87</td>\n",
       "      <td>0.91</td>\n",
       "      <td>0.91</td>\n",
       "      <td>0.87</td>\n",
       "      <td>0.84</td>\n",
       "      <td>0.87</td>\n",
       "      <td>0.90</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "prompt          None_None  None_kojima-01  None_zhou-01  qa-10_None  \\\n",
       "dataset                                                               \n",
       "commonsense_qa       0.74            0.75          0.78        0.78   \n",
       "med_qa               0.38            0.39          0.40        0.36   \n",
       "medmc_qa             0.46            0.44          0.54        0.48   \n",
       "open_book_qa         0.74            0.71          0.81        0.74   \n",
       "strategy_qa          0.64            0.73          0.67        0.66   \n",
       "worldtree            0.88            0.85          0.87        0.87   \n",
       "\n",
       "prompt          qa-12_None  qa-13_None  qa-16_None  qa-17_None  refl-01_None  \\\n",
       "dataset                                                                        \n",
       "commonsense_qa        0.78        0.80        0.75        0.71          0.79   \n",
       "med_qa                0.39        0.38        0.41        0.37          0.37   \n",
       "medmc_qa              0.52        0.49        0.53        0.51          0.44   \n",
       "open_book_qa          0.76        0.71        0.74        0.73          0.75   \n",
       "strategy_qa           0.66        0.72        0.68        0.66          0.62   \n",
       "worldtree             0.91        0.91        0.87        0.84          0.87   \n",
       "\n",
       "prompt          zhou-01-ins_None  \n",
       "dataset                           \n",
       "commonsense_qa              0.79  \n",
       "med_qa                      0.37  \n",
       "medmc_qa                    0.45  \n",
       "open_book_qa                0.74  \n",
       "strategy_qa                 0.66  \n",
       "worldtree                   0.90  "
      ]
     },
     "execution_count": 273,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc_dataset_prompt_mean, acc_dataset_prompt_std, acc_dataset_prompt_ci_lower, acc_dataset_prompt_ci_upper = calculate_confidence_intervals(df, \"dataset\", \"prompt\", \"correct_answer\", confidence_level=0.95)\n",
    "acc_dataset_prompt_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pandas does not work to compute the mean if there is None in the the column, I have no clue why\n",
    "# It works when the column \"correct_answer\" is not defined as above. Also just works in pandas 1.5, not in 2.x\n",
    "# acc_model_prompt = df.groupby([\"model\", \"prompt\"]).mean(\"correct_answer\").pivot_table(index=\"model\", columns=\"prompt\", values=\"correct_answer\")\n",
    "\n",
    "# acc_model_prompt = df.groupby([\"model\", \"prompt\"]).mean().pivot_table(index=\"model\", columns=\"prompt\", values=\"correct_answer\")\n",
    "# acc_model_prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_confidence_intervals_simple_tables(dataframe, index_col, values_col, confidence_level=0.95):\n",
    "    df = dataframe.copy()\n",
    "    df = df.dropna(subset=['correct_answer'])\n",
    "    df[\"correct_answer\"] = df[\"correct_answer\"].fillna(np.nan)\n",
    "    from scipy.stats import t\n",
    "    # Calculate mean accuracy scores\n",
    "    acc = df.groupby([index_col]).mean()\n",
    "\n",
    "    # Calculate standard error for each group\n",
    "    standard_errors = df.groupby([index_col]).sem().pivot_table(index=index_col, values=values_col)\n",
    "\n",
    "    # Calculate standard deviations for each group\n",
    "    standard_deviations = df.groupby([index_col]).std().pivot_table(index=index_col, values=values_col)\n",
    "\n",
    "    # Calculate degrees of freedom for each group\n",
    "    degrees_freedom = df.groupby([index_col]).count().pivot_table(index=index_col, values=values_col) - 1\n",
    "\n",
    "    # Calculate the t-score for the chosen confidence level\n",
    "    t_scores = t.ppf((1 + confidence_level) / 2, degrees_freedom)\n",
    "\n",
    "    # Calculate the margin of error for each group\n",
    "    margin_of_error = t_scores * standard_errors\n",
    "\n",
    "    # Calculate the confidence intervals\n",
    "    lower_bound = acc - margin_of_error\n",
    "    upper_bound = acc + margin_of_error\n",
    "\n",
    "    acc[\"std\"] = standard_deviations\n",
    "    acc[\"ci_lower\"] = lower_bound\n",
    "    acc[\"ci_upper\"] = upper_bound\n",
    "\n",
    "    acc.rename(columns={values_col: \"mean\"}, inplace=True)\n",
    "\n",
    "    acc.reset_index(inplace=True)\n",
    "\n",
    "    return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1153/1312718999.py:7: FutureWarning: The default value of numeric_only in DataFrameGroupBy.mean is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  acc = df.groupby([index_col]).mean()\n",
      "/tmp/ipykernel_1153/1312718999.py:10: FutureWarning: The default value of numeric_only in DataFrameGroupBy.sem is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  standard_errors = df.groupby([index_col]).sem().pivot_table(index=index_col, values=values_col)\n",
      "/tmp/ipykernel_1153/1312718999.py:13: FutureWarning: The default value of numeric_only in DataFrameGroupBy.std is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  standard_deviations = df.groupby([index_col]).std().pivot_table(index=index_col, values=values_col)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>ci_lower</th>\n",
       "      <th>ci_upper</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>commonsense_qa</td>\n",
       "      <td>0.77</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>med_qa</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.49</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>medmc_qa</td>\n",
       "      <td>0.49</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>open_book_qa</td>\n",
       "      <td>0.74</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.72</td>\n",
       "      <td>0.76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>strategy_qa</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.47</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>worldtree</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.33</td>\n",
       "      <td>0.86</td>\n",
       "      <td>0.89</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          dataset  mean  std  ci_lower  ci_upper\n",
       "0  commonsense_qa  0.77 0.42      0.75      0.79\n",
       "1          med_qa  0.38 0.49      0.36      0.40\n",
       "2        medmc_qa  0.49 0.50      0.46      0.51\n",
       "3    open_book_qa  0.74 0.44      0.72      0.76\n",
       "4     strategy_qa  0.67 0.47      0.65      0.69\n",
       "5       worldtree  0.88 0.33      0.86      0.89"
      ]
     },
     "execution_count": 276,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc_dataset = calculate_confidence_intervals_simple_tables(df, \"dataset\", \"correct_answer\", confidence_level=0.95)\n",
    "acc_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1153/1312718999.py:7: FutureWarning: The default value of numeric_only in DataFrameGroupBy.mean is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  acc = df.groupby([index_col]).mean()\n",
      "/tmp/ipykernel_1153/1312718999.py:10: FutureWarning: The default value of numeric_only in DataFrameGroupBy.sem is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  standard_errors = df.groupby([index_col]).sem().pivot_table(index=index_col, values=values_col)\n",
      "/tmp/ipykernel_1153/1312718999.py:13: FutureWarning: The default value of numeric_only in DataFrameGroupBy.std is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  standard_deviations = df.groupby([index_col]).std().pivot_table(index=index_col, values=values_col)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>ci_lower</th>\n",
       "      <th>ci_upper</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>command-xlarge-nightly</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>flan-T5-xxl</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.49</td>\n",
       "      <td>0.59</td>\n",
       "      <td>0.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gpt-3.5-turbo</td>\n",
       "      <td>0.74</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.72</td>\n",
       "      <td>0.76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gpt-4</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.83</td>\n",
       "      <td>0.86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>0.59</td>\n",
       "      <td>0.49</td>\n",
       "      <td>0.56</td>\n",
       "      <td>0.61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>text-davinci-003</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.65</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    model  mean  std  ci_lower  ci_upper\n",
       "0  command-xlarge-nightly  0.52 0.50      0.50      0.55\n",
       "1             flan-T5-xxl  0.61 0.49      0.59      0.63\n",
       "2           gpt-3.5-turbo  0.74 0.44      0.72      0.76\n",
       "3                   gpt-4  0.85 0.36      0.83      0.86\n",
       "4        text-davinci-002  0.59 0.49      0.56      0.61\n",
       "5        text-davinci-003  0.63 0.48      0.61      0.65"
      ]
     },
     "execution_count": 277,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc_model = calculate_confidence_intervals_simple_tables(df, \"model\", \"correct_answer\", confidence_level=0.95)\n",
    "acc_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1153/1312718999.py:7: FutureWarning: The default value of numeric_only in DataFrameGroupBy.mean is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  acc = df.groupby([index_col]).mean()\n",
      "/tmp/ipykernel_1153/1312718999.py:10: FutureWarning: The default value of numeric_only in DataFrameGroupBy.sem is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  standard_errors = df.groupby([index_col]).sem().pivot_table(index=index_col, values=values_col)\n",
      "/tmp/ipykernel_1153/1312718999.py:13: FutureWarning: The default value of numeric_only in DataFrameGroupBy.std is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  standard_deviations = df.groupby([index_col]).std().pivot_table(index=index_col, values=values_col)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prompt</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>ci_lower</th>\n",
       "      <th>ci_upper</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>None_None</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>None_kojima-01</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>None_zhou-01</td>\n",
       "      <td>0.68</td>\n",
       "      <td>0.47</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>qa-10_None</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>qa-12_None</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.47</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>qa-13_None</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.47</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>qa-16_None</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.47</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>qa-17_None</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>refl-01_None</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>zhou-01-ins_None</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.68</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             prompt  mean  std  ci_lower  ci_upper\n",
       "0         None_None  0.64 0.48      0.61      0.67\n",
       "1    None_kojima-01  0.64 0.48      0.62      0.67\n",
       "2      None_zhou-01  0.68 0.47      0.65      0.70\n",
       "3        qa-10_None  0.65 0.48      0.62      0.68\n",
       "4        qa-12_None  0.67 0.47      0.64      0.70\n",
       "5        qa-13_None  0.67 0.47      0.64      0.69\n",
       "6        qa-16_None  0.66 0.47      0.63      0.69\n",
       "7        qa-17_None  0.64 0.48      0.61      0.66\n",
       "8      refl-01_None  0.64 0.48      0.61      0.67\n",
       "9  zhou-01-ins_None  0.65 0.48      0.63      0.68"
      ]
     },
     "execution_count": 278,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc_prompt = calculate_confidence_intervals_simple_tables(df, \"prompt\", \"correct_answer\", confidence_level=0.95)\n",
    "acc_prompt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Krippendorff's alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [],
   "source": [
    "# KRIPPENDORFF ALPHA all datasets together, pooling over datasets\n",
    "# Define bootstrapping parameters\n",
    "n_bootstraps = 1000\n",
    "confidence_level = 0.95\n",
    "np.random.seed(42)  # Set the seed for reproducibility\n",
    "random.seed(42)\n",
    "\n",
    "# Perform bootstrapping and calculate means and confidence intervals\n",
    "results = []\n",
    "\n",
    "for (model, prompt), group in df.groupby(['model', 'prompt']):\n",
    "    unique_datasets = group['dataset'].unique()\n",
    "    dataset_means = {dataset: [] for dataset in unique_datasets}\n",
    "\n",
    "    for _ in range(n_bootstraps):\n",
    "        random_dataset = random.choice(unique_datasets)\n",
    "        random_group = group[group['dataset'] == random_dataset]\n",
    "\n",
    "        answer_label = random_group['answer_label'].sample(n=len(random_group), replace=True)\n",
    "        indices = answer_label.index\n",
    "        answer_from_choices = random_group.loc[indices, 'answer_from_choices']\n",
    "\n",
    "        metric = krippendorff.alpha(reliability_data=[list(answer_label), list(answer_from_choices)], level_of_measurement='nominal')\n",
    "        # there seem to be negative values, which are corrected to zero\n",
    "        if metric < 0:\n",
    "            metric = 0\n",
    "        dataset_means[random_dataset].append(metric)\n",
    "\n",
    "    all_means = np.concatenate(list(dataset_means.values()))\n",
    "    mean = np.mean(all_means)\n",
    "\n",
    "    # Calculate the pooled standard deviation\n",
    "    dataset_stds = {dataset: np.std(means) for dataset, means in dataset_means.items()}\n",
    "\n",
    "    squared_stds = [std ** 2 for std in dataset_stds.values()]\n",
    "    pooled_std_dev = np.sqrt(sum(squared_stds) / len(squared_stds))\n",
    "\n",
    "    # Calculate the margin of error using the z-score\n",
    "    z_score = scipy.stats.norm.ppf((1 + confidence_level) / 2)\n",
    "    margin_of_error = z_score * (pooled_std_dev / np.sqrt(len(unique_datasets)))\n",
    "\n",
    "    # Compute the confidence interval\n",
    "    ci_lower = mean - margin_of_error\n",
    "    ci_upper = mean + margin_of_error\n",
    "\n",
    "    results.append({\n",
    "        'model': model,\n",
    "        'prompt': prompt,\n",
    "        'mean': mean,\n",
    "        'std': pooled_std_dev,\n",
    "        'ci_lower': ci_lower,\n",
    "        'ci_upper': ci_upper\n",
    "    })\n",
    "\n",
    "# Convert the results to a DataFrame\n",
    "results_df = pd.DataFrame(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>prompt</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>ci_lower</th>\n",
       "      <th>ci_upper</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>command-xlarge-nightly</td>\n",
       "      <td>None_None</td>\n",
       "      <td>0.26</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.18</td>\n",
       "      <td>0.33</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    model     prompt  mean  std  ci_lower  ci_upper\n",
       "0  command-xlarge-nightly  None_None  0.26 0.09      0.18      0.33"
      ]
     },
     "execution_count": 257,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kripp_model_prompt = results_df.copy()\n",
    "kripp_model_prompt.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>prompt</th>\n",
       "      <th>None_None</th>\n",
       "      <th>None_kojima-01</th>\n",
       "      <th>None_zhou-01</th>\n",
       "      <th>qa-10_None</th>\n",
       "      <th>qa-12_None</th>\n",
       "      <th>qa-13_None</th>\n",
       "      <th>qa-16_None</th>\n",
       "      <th>qa-17_None</th>\n",
       "      <th>refl-01_None</th>\n",
       "      <th>zhou-01-ins_None</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>command-xlarge-nightly</th>\n",
       "      <td>0.26</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.34</td>\n",
       "      <td>0.33</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.34</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>flan-T5-xxl</th>\n",
       "      <td>0.49</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.41</td>\n",
       "      <td>0.43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gpt-3.5-turbo</th>\n",
       "      <td>0.61</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.59</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gpt-4</th>\n",
       "      <td>0.71</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.83</td>\n",
       "      <td>0.77</td>\n",
       "      <td>0.79</td>\n",
       "      <td>0.78</td>\n",
       "      <td>0.77</td>\n",
       "      <td>0.74</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>text-davinci-002</th>\n",
       "      <td>0.41</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.53</td>\n",
       "      <td>0.37</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>text-davinci-003</th>\n",
       "      <td>0.44</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.43</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.49</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "prompt                  None_None  None_kojima-01  None_zhou-01  qa-10_None  \\\n",
       "model                                                                         \n",
       "command-xlarge-nightly       0.26            0.25          0.35        0.34   \n",
       "flan-T5-xxl                  0.49            0.46          0.44        0.45   \n",
       "gpt-3.5-turbo                0.61            0.66          0.62        0.61   \n",
       "gpt-4                        0.71            0.80          0.83        0.77   \n",
       "text-davinci-002             0.41            0.42          0.53        0.37   \n",
       "text-davinci-003             0.44            0.45          0.50        0.46   \n",
       "\n",
       "prompt                  qa-12_None  qa-13_None  qa-16_None  qa-17_None  \\\n",
       "model                                                                    \n",
       "command-xlarge-nightly        0.33        0.42        0.34        0.31   \n",
       "flan-T5-xxl                   0.50        0.46        0.42        0.44   \n",
       "gpt-3.5-turbo                 0.59        0.61        0.61        0.58   \n",
       "gpt-4                         0.79        0.78        0.77        0.74   \n",
       "text-davinci-002              0.44        0.44        0.51        0.35   \n",
       "text-davinci-003              0.52        0.46        0.43        0.46   \n",
       "\n",
       "prompt                  refl-01_None  zhou-01-ins_None  \n",
       "model                                                   \n",
       "command-xlarge-nightly          0.32              0.38  \n",
       "flan-T5-xxl                     0.41              0.43  \n",
       "gpt-3.5-turbo                   0.58              0.64  \n",
       "gpt-4                           0.76              0.79  \n",
       "text-davinci-002                0.38              0.33  \n",
       "text-davinci-003                0.48              0.49  "
      ]
     },
     "execution_count": 258,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kripp_model_prompt_mean = kripp_model_prompt.pivot_table(values='mean', index='model', columns='prompt')\n",
    "kripp_model_prompt_std = kripp_model_prompt.pivot_table(values='std', index='model', columns='prompt')\n",
    "kripp_model_prompt_ci_lower = kripp_model_prompt.pivot_table(values='ci_lower', index='model', columns='prompt')\n",
    "kripp_model_prompt_ci_upper = kripp_model_prompt.pivot_table(values='ci_upper', index='model', columns='prompt')\n",
    "kripp_model_prompt_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>prompt</th>\n",
       "      <th>None_None</th>\n",
       "      <th>None_kojima-01</th>\n",
       "      <th>None_zhou-01</th>\n",
       "      <th>qa-10_None</th>\n",
       "      <th>qa-12_None</th>\n",
       "      <th>qa-13_None</th>\n",
       "      <th>qa-16_None</th>\n",
       "      <th>qa-17_None</th>\n",
       "      <th>refl-01_None</th>\n",
       "      <th>zhou-01-ins_None</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>command-xlarge-nightly</th>\n",
       "      <td>0.09</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>flan-T5-xxl</th>\n",
       "      <td>0.11</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gpt-3.5-turbo</th>\n",
       "      <td>0.10</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gpt-4</th>\n",
       "      <td>0.10</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>text-davinci-002</th>\n",
       "      <td>0.11</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>text-davinci-003</th>\n",
       "      <td>0.11</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "prompt                  None_None  None_kojima-01  None_zhou-01  qa-10_None  \\\n",
       "model                                                                         \n",
       "command-xlarge-nightly       0.09            0.11          0.10        0.11   \n",
       "flan-T5-xxl                  0.11            0.10          0.08        0.10   \n",
       "gpt-3.5-turbo                0.10            0.11          0.11        0.11   \n",
       "gpt-4                        0.10            0.09          0.08        0.08   \n",
       "text-davinci-002             0.11            0.11          0.11        0.09   \n",
       "text-davinci-003             0.11            0.11          0.11        0.11   \n",
       "\n",
       "prompt                  qa-12_None  qa-13_None  qa-16_None  qa-17_None  \\\n",
       "model                                                                    \n",
       "command-xlarge-nightly        0.09        0.11        0.10        0.11   \n",
       "flan-T5-xxl                   0.10        0.10        0.11        0.11   \n",
       "gpt-3.5-turbo                 0.12        0.11        0.12        0.11   \n",
       "gpt-4                         0.09        0.09        0.09        0.10   \n",
       "text-davinci-002              0.11        0.11        0.12        0.10   \n",
       "text-davinci-003              0.11        0.11        0.10        0.10   \n",
       "\n",
       "prompt                  refl-01_None  zhou-01-ins_None  \n",
       "model                                                   \n",
       "command-xlarge-nightly          0.08              0.10  \n",
       "flan-T5-xxl                     0.08              0.10  \n",
       "gpt-3.5-turbo                   0.12              0.12  \n",
       "gpt-4                           0.10              0.09  \n",
       "text-davinci-002                0.11              0.08  \n",
       "text-davinci-003                0.11              0.11  "
      ]
     },
     "execution_count": 259,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kripp_model_prompt_std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [],
   "source": [
    "# acc_model_prompt/kripp_model_prompt.pivot_table(index=\"model\", columns=\"prompt\", values=\"mean\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {},
   "outputs": [],
   "source": [
    "# kripp_model_prompt_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {},
   "outputs": [],
   "source": [
    "# acc_model_prompt_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {},
   "outputs": [],
   "source": [
    "# should be correct, except in very similar scores, cause there can be small differences in the bootstrapping\n",
    "# kripp_model_prompt_mean.rank(1) == acc_model_prompt_mean.rank(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [],
   "source": [
    "# KRIPPENDORFF ALPHA dataset by prompt sum over models\n",
    "\n",
    "# Define bootstrapping parameters\n",
    "n_bootstraps = 1000\n",
    "confidence_level = 0.95\n",
    "np.random.seed(42)  # Set the seed for reproducibility\n",
    "random.seed(42)\n",
    "\n",
    "# Perform bootstrapping and calculate means and confidence intervals\n",
    "results = []\n",
    "\n",
    "for (dataset, prompt), group in df.groupby(['dataset', 'prompt']):\n",
    "    bootstrapped_means = []\n",
    "    for _ in range(n_bootstraps):\n",
    "        answer_label = group['answer_label'].sample(n=len(group), replace=True)\n",
    "        indices = answer_label.index\n",
    "        answer_from_choices = group.loc[indices, 'answer_from_choices']\n",
    "\n",
    "        metric = krippendorff.alpha(reliability_data=[list(answer_label), list(answer_from_choices)],level_of_measurement='nominal')\n",
    "        if metric < 0:\n",
    "            metric = 0\n",
    "        bootstrapped_means.append(metric)\n",
    "    \n",
    "    mean = np.mean(bootstrapped_means)\n",
    "    std = np.std(bootstrapped_means)\n",
    "    lower = np.percentile(bootstrapped_means, (1 - confidence_level) / 2 * 100)\n",
    "    upper = np.percentile(bootstrapped_means, (1 + confidence_level) / 2 * 100)\n",
    "    \n",
    "    results.append({\n",
    "        'dataset': dataset,\n",
    "        # 'model': model,\n",
    "        'prompt': prompt,\n",
    "        'mean': mean,\n",
    "        'std': std,\n",
    "        'ci_lower': lower,\n",
    "        'ci_upper': upper\n",
    "    })\n",
    "\n",
    "# Convert the results to a DataFrame and pivot\n",
    "results_df = pd.DataFrame(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>prompt</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>ci_lower</th>\n",
       "      <th>ci_upper</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>commonsense_qa</td>\n",
       "      <td>None_None</td>\n",
       "      <td>0.68</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.76</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          dataset     prompt  mean  std  ci_lower  ci_upper\n",
       "0  commonsense_qa  None_None  0.68 0.04      0.60      0.76"
      ]
     },
     "execution_count": 250,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kripp_dataset_prompt = results_df.copy()\n",
    "kripp_dataset_prompt.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>prompt</th>\n",
       "      <th>None_None</th>\n",
       "      <th>None_kojima-01</th>\n",
       "      <th>None_zhou-01</th>\n",
       "      <th>qa-10_None</th>\n",
       "      <th>qa-12_None</th>\n",
       "      <th>qa-13_None</th>\n",
       "      <th>qa-16_None</th>\n",
       "      <th>qa-17_None</th>\n",
       "      <th>refl-01_None</th>\n",
       "      <th>zhou-01-ins_None</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>commonsense_qa</th>\n",
       "      <td>0.68</td>\n",
       "      <td>0.69</td>\n",
       "      <td>0.72</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.72</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.68</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>med_qa</th>\n",
       "      <td>0.21</td>\n",
       "      <td>0.22</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.19</td>\n",
       "      <td>0.22</td>\n",
       "      <td>0.21</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.19</td>\n",
       "      <td>0.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>medmc_qa</th>\n",
       "      <td>0.28</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.37</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>open_book_qa</th>\n",
       "      <td>0.65</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.74</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>strategy_qa</th>\n",
       "      <td>0.24</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.33</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>worldtree</th>\n",
       "      <td>0.84</td>\n",
       "      <td>0.79</td>\n",
       "      <td>0.83</td>\n",
       "      <td>0.82</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.87</td>\n",
       "      <td>0.82</td>\n",
       "      <td>0.78</td>\n",
       "      <td>0.82</td>\n",
       "      <td>0.86</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "prompt          None_None  None_kojima-01  None_zhou-01  qa-10_None  \\\n",
       "dataset                                                               \n",
       "commonsense_qa       0.68            0.69          0.72        0.73   \n",
       "med_qa               0.21            0.22          0.23        0.19   \n",
       "medmc_qa             0.28            0.25          0.37        0.30   \n",
       "open_book_qa         0.65            0.61          0.74        0.65   \n",
       "strategy_qa          0.24            0.46          0.32        0.27   \n",
       "worldtree            0.84            0.79          0.83        0.82   \n",
       "\n",
       "prompt          qa-12_None  qa-13_None  qa-16_None  qa-17_None  refl-01_None  \\\n",
       "dataset                                                                        \n",
       "commonsense_qa        0.72        0.75        0.68        0.63          0.73   \n",
       "med_qa                0.22        0.21        0.25        0.20          0.19   \n",
       "medmc_qa              0.35        0.31        0.36        0.32          0.25   \n",
       "open_book_qa          0.67        0.61        0.64        0.63          0.66   \n",
       "strategy_qa           0.27        0.42        0.33        0.30          0.23   \n",
       "worldtree             0.88        0.87        0.82        0.78          0.82   \n",
       "\n",
       "prompt          zhou-01-ins_None  \n",
       "dataset                           \n",
       "commonsense_qa              0.73  \n",
       "med_qa                      0.19  \n",
       "medmc_qa                    0.26  \n",
       "open_book_qa                0.65  \n",
       "strategy_qa                 0.28  \n",
       "worldtree                   0.86  "
      ]
     },
     "execution_count": 251,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kripp_dataset_prompt_mean = kripp_dataset_prompt.pivot_table(values='mean', index='dataset', columns='prompt')\n",
    "kripp_dataset_prompt_std = kripp_dataset_prompt.pivot_table(values='std', index='dataset', columns='prompt')\n",
    "kripp_dataset_prompt_ci_lower = kripp_dataset_prompt.pivot_table(values='ci_lower', index='dataset', columns='prompt')\n",
    "kripp_dataset_prompt_ci_upper = kripp_dataset_prompt.pivot_table(values='ci_upper', index='dataset', columns='prompt')\n",
    "kripp_dataset_prompt_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [],
   "source": [
    "# kripp_dataset_prompt_mean.std(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [],
   "source": [
    "# kripp_dataset_prompt_mean.std(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [],
   "source": [
    "# acc_dataset_prompt/kripp_dataset_prompt.pivot_table(values='mean', index='dataset', columns='prompt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {},
   "outputs": [],
   "source": [
    "# kripp_dataset_prompt_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {},
   "outputs": [],
   "source": [
    "# acc_dataset_prompt_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "metadata": {},
   "outputs": [],
   "source": [
    "# should be correct, except in very similar scores, cause there can be small differences in the bootstrapping\n",
    "# kripp_dataset_prompt_mean.rank(1) == acc_dataset_prompt_mean.rank(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [],
   "source": [
    "# KRIPPENDORFF ALPHA dataset by model, sum over prompts\n",
    "# Define bootstrapping parameters\n",
    "n_bootstraps = 1000\n",
    "confidence_level = 0.95\n",
    "np.random.seed(42)  # Set the seed for reproducibility\n",
    "random.seed(42)\n",
    "\n",
    "# Perform bootstrapping and calculate means and confidence intervals\n",
    "results = []\n",
    "\n",
    "for (dataset, model), group in df.groupby(['dataset', 'model']):\n",
    "    bootstrapped_means = []\n",
    "    for _ in range(n_bootstraps):\n",
    "        answer_label = group['answer_label'].sample(n=len(group), replace=True)\n",
    "        indices = answer_label.index\n",
    "        answer_from_choices = group.loc[indices, 'answer_from_choices']\n",
    "\n",
    "        metric = krippendorff.alpha(reliability_data=[list(answer_label), list(answer_from_choices)],level_of_measurement='nominal')\n",
    "        if metric < 0:\n",
    "            metric = 0\n",
    "        bootstrapped_means.append(metric)\n",
    "    \n",
    "    mean = np.mean(bootstrapped_means)\n",
    "    std = np.std(bootstrapped_means)\n",
    "    lower = np.percentile(bootstrapped_means, (1 - confidence_level) / 2 * 100)\n",
    "    upper = np.percentile(bootstrapped_means, (1 + confidence_level) / 2 * 100)\n",
    "    \n",
    "    results.append({\n",
    "        'dataset': dataset,\n",
    "        'model': model,\n",
    "        # 'prompt': prompt,\n",
    "        'std': std,\n",
    "        'mean': mean,\n",
    "        'ci_lower': lower,\n",
    "        'ci_upper': upper\n",
    "    })\n",
    "\n",
    "# Convert the results to a DataFrame and pivot\n",
    "results_df = pd.DataFrame(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>model</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>ci_lower</th>\n",
       "      <th>ci_upper</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>commonsense_qa</td>\n",
       "      <td>command-xlarge-nightly</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.57</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.64</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          dataset                   model  std  mean  ci_lower  ci_upper\n",
       "0  commonsense_qa  command-xlarge-nightly 0.03  0.57      0.50      0.64"
      ]
     },
     "execution_count": 241,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kripp_dataset_model = results_df.copy()\n",
    "kripp_dataset_model.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>model</th>\n",
       "      <th>command-xlarge-nightly</th>\n",
       "      <th>flan-T5-xxl</th>\n",
       "      <th>gpt-3.5-turbo</th>\n",
       "      <th>gpt-4</th>\n",
       "      <th>text-davinci-002</th>\n",
       "      <th>text-davinci-003</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>commonsense_qa</th>\n",
       "      <td>0.57</td>\n",
       "      <td>0.81</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.82</td>\n",
       "      <td>0.68</td>\n",
       "      <td>0.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>med_qa</th>\n",
       "      <td>0.06</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.55</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>medmc_qa</th>\n",
       "      <td>0.08</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>open_book_qa</th>\n",
       "      <td>0.43</td>\n",
       "      <td>0.69</td>\n",
       "      <td>0.77</td>\n",
       "      <td>0.91</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>strategy_qa</th>\n",
       "      <td>0.10</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.69</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>worldtree</th>\n",
       "      <td>0.67</td>\n",
       "      <td>0.77</td>\n",
       "      <td>0.89</td>\n",
       "      <td>0.97</td>\n",
       "      <td>0.84</td>\n",
       "      <td>0.84</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "model           command-xlarge-nightly  flan-T5-xxl  gpt-3.5-turbo  gpt-4  \\\n",
       "dataset                                                                     \n",
       "commonsense_qa                    0.57         0.81           0.70   0.82   \n",
       "med_qa                            0.06         0.02           0.40   0.55   \n",
       "medmc_qa                          0.08         0.10           0.51   0.73   \n",
       "open_book_qa                      0.43         0.69           0.77   0.91   \n",
       "strategy_qa                       0.10         0.23           0.44   0.69   \n",
       "worldtree                         0.67         0.77           0.89   0.97   \n",
       "\n",
       "model           text-davinci-002  text-davinci-003  \n",
       "dataset                                             \n",
       "commonsense_qa              0.68              0.68  \n",
       "med_qa                      0.09              0.17  \n",
       "medmc_qa                    0.20              0.21  \n",
       "open_book_qa                0.45              0.66  \n",
       "strategy_qa                 0.20              0.22  \n",
       "worldtree                   0.84              0.84  "
      ]
     },
     "execution_count": 242,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kripp_dataset_model_mean = kripp_dataset_model.pivot_table(values='mean', index='dataset', columns='model')\n",
    "kripp_dataset_model_std = kripp_dataset_model.pivot_table(values='std', index='dataset', columns='model')\n",
    "kripp_dataset_model_ci_lower = kripp_dataset_model.pivot_table(values='ci_lower', index='dataset', columns='model')\n",
    "kripp_dataset_model_ci_upper = kripp_dataset_model.pivot_table(values='ci_upper', index='dataset', columns='model')\n",
    "kripp_dataset_model_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "metadata": {},
   "outputs": [],
   "source": [
    "# acc_dataset_model_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {},
   "outputs": [],
   "source": [
    "# should be correct, except in very similar scores, cause there can be small differences in the bootstrapping\n",
    "# kripp_dataset_model_mean.rank(1) == acc_dataset_model_mean.rank(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# acc_dataset_model/kripp_dataset_model.pivot_table(values='mean', index='dataset', columns='model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [],
   "source": [
    "# KRIPPENDORFF ALPHA only DATASET, sum over prompts and models\n",
    "# Define bootstrapping parameters\n",
    "n_bootstraps = 1000\n",
    "confidence_level = 0.95\n",
    "np.random.seed(42)  # Set the seed for reproducibility\n",
    "random.seed(42)\n",
    "\n",
    "# Perform bootstrapping and calculate means and confidence intervals\n",
    "results = []\n",
    "\n",
    "for dataset, group in df.groupby('dataset'):\n",
    "    bootstrapped_means = []\n",
    "    for _ in range(n_bootstraps):\n",
    "        answer_label = group['answer_label'].sample(n=len(group), replace=True)\n",
    "        indices = answer_label.index\n",
    "        answer_from_choices = group.loc[indices, 'answer_from_choices']\n",
    "\n",
    "        metric = krippendorff.alpha(reliability_data=[list(answer_label), list(answer_from_choices)],level_of_measurement='nominal')\n",
    "        if metric < 0:\n",
    "            metric = 0\n",
    "        bootstrapped_means.append(metric)\n",
    "    \n",
    "    mean = np.mean(bootstrapped_means)\n",
    "    std = np.std(bootstrapped_means)\n",
    "    lower = np.percentile(bootstrapped_means, (1 - confidence_level) / 2 * 100)\n",
    "    upper = np.percentile(bootstrapped_means, (1 + confidence_level) / 2 * 100)\n",
    "    \n",
    "    results.append({\n",
    "        'dataset': dataset,\n",
    "        # 'model': model,\n",
    "        # 'prompt': prompt,\n",
    "        'mean': mean,\n",
    "        'std': std,\n",
    "        'ci_lower': lower,\n",
    "        'ci_upper': upper\n",
    "    })\n",
    "\n",
    "# Convert the results to a DataFrame and pivot\n",
    "results_df = pd.DataFrame(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [],
   "source": [
    "kripp_dataset = results_df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>ci_lower</th>\n",
       "      <th>ci_upper</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>commonsense_qa</td>\n",
       "      <td>0.71</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.68</td>\n",
       "      <td>0.73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>med_qa</td>\n",
       "      <td>0.21</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.19</td>\n",
       "      <td>0.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>medmc_qa</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.28</td>\n",
       "      <td>0.34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>open_book_qa</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>strategy_qa</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>worldtree</td>\n",
       "      <td>0.83</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.81</td>\n",
       "      <td>0.85</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          dataset  mean  std  ci_lower  ci_upper\n",
       "0  commonsense_qa  0.71 0.01      0.68      0.73\n",
       "1          med_qa  0.21 0.01      0.19      0.24\n",
       "2        medmc_qa  0.31 0.02      0.28      0.34\n",
       "3    open_book_qa  0.65 0.01      0.63      0.68\n",
       "4     strategy_qa  0.31 0.02      0.27      0.36\n",
       "5       worldtree  0.83 0.01      0.81      0.85"
      ]
     },
     "execution_count": 286,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kripp_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {},
   "outputs": [],
   "source": [
    "# KRIPPENDORFF ALPHA model scores by summing over prompts and datasets\n",
    "# BOOTSTRAPPING SEPERATELY FOR EACH DATASET, SINCE THE NUMBER OF CHOICES IS DIFFERENT\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "import scipy.stats\n",
    "\n",
    "n_bootstraps = 1000\n",
    "confidence_level = 0.95\n",
    "np.random.seed(42)  # Set the seed for reproducibility\n",
    "random.seed(42)\n",
    "\n",
    "# Perform bootstrapping and calculate means and confidence intervals\n",
    "results = []\n",
    "\n",
    "for model, group in df.groupby('model'):\n",
    "    unique_datasets = group['dataset'].unique()\n",
    "    dataset_means = {dataset: [] for dataset in unique_datasets}\n",
    "\n",
    "    for _ in range(n_bootstraps):\n",
    "\n",
    "        random_dataset = random.choice(unique_datasets)\n",
    "        random_group = group[group['dataset'] == random_dataset]\n",
    "\n",
    "        answer_label = random_group['answer_label'].sample(n=len(random_group), replace=True)\n",
    "        indices = answer_label.index\n",
    "        answer_from_choices = random_group.loc[indices, 'answer_from_choices']\n",
    "\n",
    "        metric = krippendorff.alpha(reliability_data=[list(answer_label), list(answer_from_choices)], level_of_measurement='nominal')\n",
    "        if metric < 0:\n",
    "            metric = 0\n",
    "        dataset_means[random_dataset].append(metric)\n",
    "\n",
    "    all_means = np.concatenate(list(dataset_means.values()))\n",
    "    mean = np.mean(all_means)\n",
    "\n",
    "    # Calculate the pooled standard deviation\n",
    "    dataset_stds = {dataset: np.std(means) for dataset, means in dataset_means.items()}\n",
    "\n",
    "    squared_stds = [std ** 2 for std in dataset_stds.values()]\n",
    "    pooled_std_dev = np.sqrt(sum(squared_stds) / len(squared_stds))\n",
    "\n",
    "    # Calculate the margin of error using the z-score\n",
    "    z_score = scipy.stats.norm.ppf((1 + confidence_level) / 2)\n",
    "    margin_of_error = z_score * (pooled_std_dev / np.sqrt(len(unique_datasets)))\n",
    "\n",
    "    # Compute the confidence interval\n",
    "    ci_lower = mean - margin_of_error\n",
    "    ci_upper = mean + margin_of_error\n",
    "\n",
    "    results.append({\n",
    "        'model': model,\n",
    "        'mean': mean,\n",
    "        'std': pooled_std_dev,\n",
    "        'ci_lower': ci_lower,\n",
    "        'ci_upper': ci_upper\n",
    "    })\n",
    "\n",
    "# Convert the results to a DataFrame\n",
    "results_df = pd.DataFrame(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {},
   "outputs": [],
   "source": [
    "kripp_model = results_df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>ci_lower</th>\n",
       "      <th>ci_upper</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>command-xlarge-nightly</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>flan-T5-xxl</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gpt-3.5-turbo</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.59</td>\n",
       "      <td>0.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gpt-4</td>\n",
       "      <td>0.78</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>0.41</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>text-davinci-003</td>\n",
       "      <td>0.47</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.50</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    model  mean  std  ci_lower  ci_upper\n",
       "0  command-xlarge-nightly  0.32 0.04      0.29      0.35\n",
       "1             flan-T5-xxl  0.45 0.03      0.42      0.47\n",
       "2           gpt-3.5-turbo  0.62 0.04      0.59      0.65\n",
       "3                   gpt-4  0.78 0.03      0.76      0.81\n",
       "4        text-davinci-002  0.41 0.04      0.38      0.44\n",
       "5        text-davinci-003  0.47 0.04      0.45      0.50"
      ]
     },
     "execution_count": 289,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kripp_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {},
   "outputs": [],
   "source": [
    "# KRIPPENDORFF ALPHA prompt scores by summing over models and datasets\n",
    "# BOOTSTRAPPING SEPERATELY FOR EACH DATASET, SINCE THE NUMBER OF CHOICES IS DIFFERENT\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "import scipy.stats\n",
    "\n",
    "n_bootstraps = 1000\n",
    "confidence_level = 0.95\n",
    "np.random.seed(42)  # Set the seed for reproducibility\n",
    "random.seed(42)\n",
    "\n",
    "# Perform bootstrapping and calculate means and confidence intervals\n",
    "results = []\n",
    "\n",
    "for prompt, group in df.groupby('prompt'):\n",
    "    unique_datasets = group['dataset'].unique()\n",
    "    dataset_means = {dataset: [] for dataset in unique_datasets}\n",
    "\n",
    "    for _ in range(n_bootstraps):\n",
    "\n",
    "        random_dataset = random.choice(unique_datasets)\n",
    "        random_group = group[group['dataset'] == random_dataset]\n",
    "\n",
    "        answer_label = random_group['answer_label'].sample(n=len(random_group), replace=True)\n",
    "        indices = answer_label.index\n",
    "        answer_from_choices = random_group.loc[indices, 'answer_from_choices']\n",
    "\n",
    "        metric = krippendorff.alpha(reliability_data=[list(answer_label), list(answer_from_choices)], level_of_measurement='nominal')\n",
    "        if metric < 0:\n",
    "            metric = 0\n",
    "        dataset_means[random_dataset].append(metric)\n",
    "\n",
    "    all_means = np.concatenate(list(dataset_means.values()))\n",
    "    mean = np.mean(all_means)\n",
    "\n",
    "    # Calculate the pooled standard deviation\n",
    "    dataset_stds = {dataset: np.std(means) for dataset, means in dataset_means.items()}\n",
    "\n",
    "    squared_stds = [std ** 2 for std in dataset_stds.values()]\n",
    "    pooled_std_dev = np.sqrt(sum(squared_stds) / len(squared_stds))\n",
    "\n",
    "    # Calculate the margin of error using the z-score\n",
    "    z_score = scipy.stats.norm.ppf((1 + confidence_level) / 2)\n",
    "    margin_of_error = z_score * (pooled_std_dev / np.sqrt(len(unique_datasets)))\n",
    "\n",
    "    # Compute the confidence interval\n",
    "    ci_lower = mean - margin_of_error\n",
    "    ci_upper = mean + margin_of_error\n",
    "\n",
    "    results.append({\n",
    "        'prompt': prompt,\n",
    "        'mean': mean,\n",
    "        'std': pooled_std_dev,\n",
    "        'ci_lower': ci_lower,\n",
    "        'ci_upper': ci_upper\n",
    "    })\n",
    "\n",
    "# Convert the results to a DataFrame\n",
    "results_df = pd.DataFrame(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {},
   "outputs": [],
   "source": [
    "kripp_prompt = results_df.copy()\n",
    "kripp_prompt.to_csv('krippendorff_alpha_prompt.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prompt</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>ci_lower</th>\n",
       "      <th>ci_upper</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>None_None</td>\n",
       "      <td>0.49</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>None_kojima-01</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.47</td>\n",
       "      <td>0.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>None_zhou-01</td>\n",
       "      <td>0.53</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>qa-10_None</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>qa-12_None</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>qa-13_None</td>\n",
       "      <td>0.54</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>qa-16_None</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.47</td>\n",
       "      <td>0.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>qa-17_None</td>\n",
       "      <td>0.47</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.43</td>\n",
       "      <td>0.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>refl-01_None</td>\n",
       "      <td>0.49</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>zhou-01-ins_None</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.54</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             prompt  mean  std  ci_lower  ci_upper\n",
       "0         None_None  0.49 0.05      0.45      0.52\n",
       "1    None_kojima-01  0.51 0.05      0.47      0.55\n",
       "2      None_zhou-01  0.53 0.05      0.50      0.57\n",
       "3        qa-10_None  0.50 0.05      0.46      0.54\n",
       "4        qa-12_None  0.52 0.05      0.48      0.56\n",
       "5        qa-13_None  0.54 0.05      0.51      0.58\n",
       "6        qa-16_None  0.51 0.05      0.47      0.55\n",
       "7        qa-17_None  0.47 0.05      0.43      0.51\n",
       "8      refl-01_None  0.49 0.05      0.45      0.53\n",
       "9  zhou-01-ins_None  0.50 0.05      0.46      0.54"
      ]
     },
     "execution_count": 292,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kripp_prompt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### creating tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "metadata": {},
   "outputs": [],
   "source": [
    "# changing the naming of prompts, models and datasets\n",
    "\n",
    "prompt_replacements = {\n",
    "    'None_None': 'Direct',\n",
    "    'qa-10_None': 'Plan',\n",
    "    'qa-12_None': 'Articulate',\n",
    "    'qa-13_None': 'Rephrase',\n",
    "    'qa-16_None': 'Elaborate',\n",
    "    'qa-17_None': 'Converse',\n",
    "    'zhou-01-ins_None': 'Zhou-instruction',\n",
    "    'refl-01_None': 'Self-critique',\n",
    "    'None_kojima-01': 'Kojima',\n",
    "    'None_zhou-01': 'Zhou'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 294,
   "metadata": {},
   "outputs": [],
   "source": [
    "# changing the naming of prompts, models and datasets\n",
    "\n",
    "model_replacements = {\n",
    "    'command-xlarge-nightly': 'Command-XL',\n",
    "    'flan-T5-xxl': 'Flan-T5-XXL',\n",
    "    'gpt-3.5-turbo': 'GPT-3.5-turbo',\n",
    "    'gpt-4': 'GPT-4',\n",
    "    'text-davinci-002': 'Davinci-002',\n",
    "    'text-davinci-003': 'Davinci-003',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "metadata": {},
   "outputs": [],
   "source": [
    "# changing the naming of prompts, models and datasets\n",
    "\n",
    "dataset_replacements = {\n",
    "    'commonsense_qa': 'CommonsenseQA',\n",
    "    'med_qa': 'MedQA',\n",
    "    'medmc_qa': 'MedMCQA',\n",
    "    'open_book_qa': 'OpenBookQA',\n",
    "    'strategy_qa': 'StrategyQA',\n",
    "    'worldtree': 'WorldTree v2',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_decimal(value):\n",
    "    if isinstance(value, float):\n",
    "        formatted_value = f'{value:.2f}'.lstrip('0')\n",
    "    else:\n",
    "        formatted_value = value\n",
    "    return formatted_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df = kripp_prompt.copy()\n",
    "# df = kripp_model.copy()\n",
    "# df = kripp_dataset.copy()\n",
    "# df = kripp_model_prompt[kripp_model_prompt.model=='gpt-4'].copy()\n",
    "# df = kripp_dataset_model.copy()\n",
    "# df = acc_prompt.copy()\n",
    "# df = acc_dataset.copy()\n",
    "df = acc_model.copy()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>ci_lower</th>\n",
       "      <th>ci_upper</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>command-xlarge-nightly</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>flan-T5-xxl</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.49</td>\n",
       "      <td>0.59</td>\n",
       "      <td>0.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gpt-3.5-turbo</td>\n",
       "      <td>0.74</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.72</td>\n",
       "      <td>0.76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gpt-4</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.83</td>\n",
       "      <td>0.86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>0.59</td>\n",
       "      <td>0.49</td>\n",
       "      <td>0.56</td>\n",
       "      <td>0.61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>text-davinci-003</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.65</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    model  mean  std  ci_lower  ci_upper\n",
       "0  command-xlarge-nightly  0.52 0.50      0.50      0.55\n",
       "1             flan-T5-xxl  0.61 0.49      0.59      0.63\n",
       "2           gpt-3.5-turbo  0.74 0.44      0.72      0.76\n",
       "3                   gpt-4  0.85 0.36      0.83      0.86\n",
       "4        text-davinci-002  0.59 0.49      0.56      0.61\n",
       "5        text-davinci-003  0.63 0.48      0.61      0.65"
      ]
     },
     "execution_count": 332,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop(columns=\"std\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {},
   "outputs": [],
   "source": [
    "if 'prompt' in df.columns:\n",
    "    df['prompt'] = df['prompt'].map(prompt_replacements)\n",
    "if 'model' in df.columns:\n",
    "    df['model'] = df['model'].map(model_replacements)\n",
    "if 'dataset' in df.columns:\n",
    "    df['dataset'] = df['dataset'].map(dataset_replacements)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.rename(columns={col: col.capitalize() for col in ['prompt', 'model', 'dataset']}, inplace=True)\n",
    "if 'mean' in df.columns:\n",
    "    df.sort_values(by='mean', inplace=True, ascending=False)\n",
    "    df.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {},
   "outputs": [],
   "source": [
    "# if isinstance(df.index, pd.RangeIndex) or pd.api.types.is_integer_dtype(df.index):\n",
    "#     df.set_index(df.columns[0], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>mean</th>\n",
       "      <th>ci_lower</th>\n",
       "      <th>ci_upper</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GPT-4</td>\n",
       "      <td>.85</td>\n",
       "      <td>.83</td>\n",
       "      <td>.86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GPT-3.5-turbo</td>\n",
       "      <td>.74</td>\n",
       "      <td>.72</td>\n",
       "      <td>.76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Davinci-003</td>\n",
       "      <td>.63</td>\n",
       "      <td>.61</td>\n",
       "      <td>.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Flan-T5-XXL</td>\n",
       "      <td>.61</td>\n",
       "      <td>.59</td>\n",
       "      <td>.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Davinci-002</td>\n",
       "      <td>.59</td>\n",
       "      <td>.56</td>\n",
       "      <td>.61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Command-XL</td>\n",
       "      <td>.52</td>\n",
       "      <td>.50</td>\n",
       "      <td>.55</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Model mean ci_lower ci_upper\n",
       "0          GPT-4  .85      .83      .86\n",
       "1  GPT-3.5-turbo  .74      .72      .76\n",
       "2    Davinci-003  .63      .61      .65\n",
       "3    Flan-T5-XXL  .61      .59      .63\n",
       "4    Davinci-002  .59      .56      .61\n",
       "5     Command-XL  .52      .50      .55"
      ]
     },
     "execution_count": 337,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.round(2)\n",
    "df = df.applymap(format_decimal)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df[\"Krippendorff's \\\\alpha\"] = df['mean'] + \" ({\\\\scriptsize\" + df['ci_lower'] + \", \" + df['ci_upper'] + \"})\"\n",
    "\n",
    "# # Drop the original 'mean', 'ci_lower', and 'ci_upper' columns\n",
    "# df = df.drop(columns=['mean', 'ci_lower', 'ci_upper'])\n",
    "\n",
    "# # Convert the DataFrame to a LaTeX table\n",
    "# latex_table = df.to_latex(column_format=\"l\" + \"c\" * len(df.columns), float_format=\"{:.2f}\".format, escape=False, index=False)\n",
    "\n",
    "# print(latex_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{table}\n",
      "\\centering\n",
      "\\begin{tabular}{lcc}\n",
      "\\hline\n",
      "        Model &                    Accuracy \\\\\n",
      "\\hline\n",
      "        GPT-4 & .85 ({\\scriptsize.83, .86}) \\\\\n",
      "GPT-3.5-turbo & .74 ({\\scriptsize.72, .76}) \\\\\n",
      "  Davinci-003 & .63 ({\\scriptsize.61, .65}) \\\\\n",
      "  Flan-T5-XXL & .61 ({\\scriptsize.59, .63}) \\\\\n",
      "  Davinci-002 & .59 ({\\scriptsize.56, .61}) \\\\\n",
      "   Command-XL & .52 ({\\scriptsize.50, .55}) \\\\\n",
      "\\hline\n",
      "\\end{tabular}\n",
      "\n",
      "\\caption{Your caption here.}\n",
      "\\label{tab:your_label}\n",
      "\\end{table}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1153/897540160.py:8: FutureWarning: In future versions `DataFrame.to_latex` is expected to utilise the base implementation of `Styler.to_latex` for formatting and rendering. The arguments signature may therefore change. It is recommended instead to use `DataFrame.style.to_latex` which also contains additional functionality.\n",
      "  latex_table = df.to_latex(column_format=\"l\" + \"c\" * len(df.columns), float_format=\"{:.2f}\".format, escape=False, index=False)\n"
     ]
    }
   ],
   "source": [
    "# df[\"Krippendorff's \\\\alpha\"] = df['mean'] + \" ({\\\\scriptsize\" + df['ci_lower'] + \", \" + df['ci_upper'] + \"})\"\n",
    "df[\"Accuracy\"] = df['mean'] + \" ({\\\\scriptsize\" + df['ci_lower'] + \", \" + df['ci_upper'] + \"})\"\n",
    "\n",
    "# Drop the original 'mean', 'ci_lower', and 'ci_upper' columns\n",
    "df = df.drop(columns=['mean', 'ci_lower', 'ci_upper'])\n",
    "\n",
    "# Create the table from the DataFrame\n",
    "latex_table = df.to_latex(column_format=\"l\" + \"c\" * len(df.columns), float_format=\"{:.2f}\".format, escape=False, index=False)\n",
    "\n",
    "# Add horizontal lines and bold text for the header row\n",
    "latex_table = latex_table.replace(\"\\\\toprule\", \"\\\\hline\")\n",
    "latex_table = latex_table.replace(\"\\\\midrule\", \"\\\\hline\")\n",
    "latex_table = latex_table.replace(\"\\\\bottomrule\", \"\\\\hline\")\n",
    "header_row = \"&\".join([f\"\\\\textbf{{{col}}}\" for col in df.columns])\n",
    "latex_table = latex_table.replace(\"&\".join(df.columns), header_row)\n",
    "\n",
    "# Wrap the table in a table environment\n",
    "latex_table = f\"\"\"\\\n",
    "\\\\begin{{table}}\n",
    "\\\\centering\n",
    "{latex_table}\n",
    "\\\\caption{{Your caption here.}}\n",
    "\\\\label{{tab:your_label}}\n",
    "\\\\end{{table}}\n",
    "\"\"\"\n",
    "\n",
    "print(latex_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_pivot = kripp_dataset_model_mean.copy().round(2).clip(0,1).applymap(format_decimal)\n",
    "ci_lower = kripp_dataset_model_ci_lower.copy().round(2).clip(0,1).applymap(format_decimal)\n",
    "ci_upper = kripp_dataset_model_ci_upper.copy().round(2).clip(0,1).applymap(format_decimal)\n",
    "\n",
    "for df in [mean_pivot, ci_lower, ci_upper]:\n",
    "    df.index = df.index.map(dataset_replacements)\n",
    "    df.columns = df.columns.map(model_replacements)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_pivot = acc_dataset_model_mean.copy().round(2).clip(0,1).applymap(format_decimal)\n",
    "ci_lower = acc_dataset_model_ci_lower.copy().round(2).clip(0,1).applymap(format_decimal)\n",
    "ci_upper = acc_dataset_model_ci_upper.copy().round(2).clip(0,1).applymap(format_decimal)\n",
    "\n",
    "for df in [mean_pivot, ci_lower, ci_upper]:\n",
    "    df.index = df.index.map(dataset_replacements)\n",
    "    df.columns = df.columns.map(model_replacements)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_pivot = kripp_dataset_prompt_mean.copy().round(2).clip(0,1).applymap(format_decimal)\n",
    "ci_lower = kripp_dataset_prompt_ci_lower.copy().round(2).clip(0,1).applymap(format_decimal)\n",
    "ci_upper = kripp_dataset_prompt_ci_upper.copy().round(2).clip(0,1).applymap(format_decimal)\n",
    "\n",
    "for df in [mean_pivot, ci_lower, ci_upper]:\n",
    "    df.index = df.index.map(dataset_replacements)\n",
    "    df.columns = df.columns.map(prompt_replacements)\n",
    "\n",
    "mean_pivot = mean_pivot.T\n",
    "ci_lower = ci_lower.T\n",
    "ci_upper = ci_upper.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_pivot = acc_dataset_prompt_mean.copy().round(2).clip(0,1).applymap(format_decimal)\n",
    "ci_lower = acc_dataset_prompt_ci_lower.copy().round(2).clip(0,1).applymap(format_decimal)\n",
    "ci_upper = acc_dataset_prompt_ci_upper.copy().round(2).clip(0,1).applymap(format_decimal)\n",
    "\n",
    "for df in [mean_pivot, ci_lower, ci_upper]:\n",
    "    df.index = df.index.map(dataset_replacements)\n",
    "    df.columns = df.columns.map(prompt_replacements)\n",
    "\n",
    "mean_pivot = mean_pivot.T\n",
    "ci_lower = ci_lower.T\n",
    "ci_upper = ci_upper.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_pivot = kripp_model_prompt_mean.copy().round(2).clip(0,1).applymap(format_decimal)\n",
    "ci_lower = kripp_model_prompt_ci_lower.copy().round(2).clip(0,1).applymap(format_decimal)\n",
    "ci_upper = kripp_model_prompt_ci_upper.copy().round(2).clip(0,1).applymap(format_decimal)\n",
    "\n",
    "for df in [mean_pivot, ci_lower, ci_upper]:\n",
    "    df.index = df.index.map(model_replacements)\n",
    "    df.columns = df.columns.map(prompt_replacements)\n",
    "\n",
    "mean_pivot = mean_pivot.T\n",
    "ci_lower = ci_lower.T\n",
    "ci_upper = ci_upper.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_pivot = acc_model_prompt_mean.copy().round(2).clip(0,1).applymap(format_decimal)\n",
    "ci_lower = acc_model_prompt_ci_lower.copy().round(2).clip(0,1).applymap(format_decimal)\n",
    "ci_upper = acc_model_prompt_ci_upper.copy().round(2).clip(0,1).applymap(format_decimal)\n",
    "\n",
    "for df in [mean_pivot, ci_lower, ci_upper]:\n",
    "    df.index = df.index.map(model_replacements)\n",
    "    df.columns = df.columns.map(prompt_replacements)\n",
    "\n",
    "mean_pivot = mean_pivot.T\n",
    "ci_lower = ci_lower.T\n",
    "ci_upper = ci_upper.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>model</th>\n",
       "      <th>Command-XL</th>\n",
       "      <th>Flan-T5-XXL</th>\n",
       "      <th>GPT-3.5-turbo</th>\n",
       "      <th>GPT-4</th>\n",
       "      <th>Davinci-002</th>\n",
       "      <th>Davinci-003</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>prompt</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Direct</th>\n",
       "      <td>.47</td>\n",
       "      <td>.62</td>\n",
       "      <td>.75</td>\n",
       "      <td>.81</td>\n",
       "      <td>.59</td>\n",
       "      <td>.61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Kojima</th>\n",
       "      <td>.45</td>\n",
       "      <td>.62</td>\n",
       "      <td>.76</td>\n",
       "      <td>.86</td>\n",
       "      <td>.56</td>\n",
       "      <td>.61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Zhou</th>\n",
       "      <td>.55</td>\n",
       "      <td>.58</td>\n",
       "      <td>.75</td>\n",
       "      <td>.89</td>\n",
       "      <td>.65</td>\n",
       "      <td>.66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Plan</th>\n",
       "      <td>.53</td>\n",
       "      <td>.62</td>\n",
       "      <td>.73</td>\n",
       "      <td>.84</td>\n",
       "      <td>.55</td>\n",
       "      <td>.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Articulate</th>\n",
       "      <td>.53</td>\n",
       "      <td>.65</td>\n",
       "      <td>.74</td>\n",
       "      <td>.86</td>\n",
       "      <td>.61</td>\n",
       "      <td>.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rephrase</th>\n",
       "      <td>.58</td>\n",
       "      <td>.62</td>\n",
       "      <td>.73</td>\n",
       "      <td>.84</td>\n",
       "      <td>.61</td>\n",
       "      <td>.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Elaborate</th>\n",
       "      <td>.55</td>\n",
       "      <td>.59</td>\n",
       "      <td>.73</td>\n",
       "      <td>.84</td>\n",
       "      <td>.65</td>\n",
       "      <td>.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Converse</th>\n",
       "      <td>.52</td>\n",
       "      <td>.61</td>\n",
       "      <td>.71</td>\n",
       "      <td>.81</td>\n",
       "      <td>.53</td>\n",
       "      <td>.62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Self-critique</th>\n",
       "      <td>.51</td>\n",
       "      <td>.58</td>\n",
       "      <td>.72</td>\n",
       "      <td>.84</td>\n",
       "      <td>.57</td>\n",
       "      <td>.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Zhou-instruction</th>\n",
       "      <td>.56</td>\n",
       "      <td>.59</td>\n",
       "      <td>.75</td>\n",
       "      <td>.85</td>\n",
       "      <td>.54</td>\n",
       "      <td>.65</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "model            Command-XL Flan-T5-XXL GPT-3.5-turbo GPT-4 Davinci-002  \\\n",
       "prompt                                                                    \n",
       "Direct                  .47         .62           .75   .81         .59   \n",
       "Kojima                  .45         .62           .76   .86         .56   \n",
       "Zhou                    .55         .58           .75   .89         .65   \n",
       "Plan                    .53         .62           .73   .84         .55   \n",
       "Articulate              .53         .65           .74   .86         .61   \n",
       "Rephrase                .58         .62           .73   .84         .61   \n",
       "Elaborate               .55         .59           .73   .84         .65   \n",
       "Converse                .52         .61           .71   .81         .53   \n",
       "Self-critique           .51         .58           .72   .84         .57   \n",
       "Zhou-instruction        .56         .59           .75   .85         .54   \n",
       "\n",
       "model            Davinci-003  \n",
       "prompt                        \n",
       "Direct                   .61  \n",
       "Kojima                   .61  \n",
       "Zhou                     .66  \n",
       "Plan                     .63  \n",
       "Articulate               .65  \n",
       "Rephrase                 .63  \n",
       "Elaborate                .60  \n",
       "Converse                 .62  \n",
       "Self-critique            .64  \n",
       "Zhou-instruction         .65  "
      ]
     },
     "execution_count": 347,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean_pivot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lcccccc}\n",
      "\\toprule\n",
      "model &                    Command-XL &                   Flan-T5-XXL &                 GPT-3.5-turbo &                         GPT-4 &                   Davinci-002 &                   Davinci-003 \\\\\n",
      "prompt           &                               &                               &                               &                               &                               &                               \\\\\n",
      "\\midrule\n",
      "Direct           &  .47 ({\\scriptsize .40, .54}) &  .62 ({\\scriptsize .55, .69}) &  .75 ({\\scriptsize .69, .81}) &  .81 ({\\scriptsize .76, .87}) &  .59 ({\\scriptsize .52, .66}) &  .61 ({\\scriptsize .54, .68}) \\\\\n",
      "Kojima           &  .45 ({\\scriptsize .38, .52}) &  .62 ({\\scriptsize .55, .69}) &  .76 ({\\scriptsize .70, .82}) &  .86 ({\\scriptsize .81, .91}) &  .56 ({\\scriptsize .49, .63}) &  .61 ({\\scriptsize .54, .68}) \\\\\n",
      "Zhou             &  .55 ({\\scriptsize .48, .62}) &  .58 ({\\scriptsize .51, .65}) &  .75 ({\\scriptsize .69, .81}) &  .89 ({\\scriptsize .84, .93}) &  .65 ({\\scriptsize .58, .71}) &  .66 ({\\scriptsize .59, .73}) \\\\\n",
      "Plan             &  .53 ({\\scriptsize .46, .60}) &  .62 ({\\scriptsize .55, .69}) &  .73 ({\\scriptsize .66, .79}) &  .84 ({\\scriptsize .79, .89}) &  .55 ({\\scriptsize .48, .62}) &  .63 ({\\scriptsize .56, .70}) \\\\\n",
      "Articulate       &  .53 ({\\scriptsize .46, .60}) &  .65 ({\\scriptsize .58, .71}) &  .74 ({\\scriptsize .67, .80}) &  .86 ({\\scriptsize .81, .91}) &  .61 ({\\scriptsize .54, .69}) &  .65 ({\\scriptsize .58, .72}) \\\\\n",
      "Rephrase         &  .58 ({\\scriptsize .51, .65}) &  .62 ({\\scriptsize .55, .69}) &  .73 ({\\scriptsize .66, .79}) &  .84 ({\\scriptsize .79, .89}) &  .61 ({\\scriptsize .54, .68}) &  .63 ({\\scriptsize .56, .70}) \\\\\n",
      "Elaborate        &  .55 ({\\scriptsize .48, .62}) &  .59 ({\\scriptsize .52, .66}) &  .73 ({\\scriptsize .66, .79}) &  .84 ({\\scriptsize .79, .89}) &  .65 ({\\scriptsize .58, .72}) &  .60 ({\\scriptsize .53, .67}) \\\\\n",
      "Converse         &  .52 ({\\scriptsize .45, .59}) &  .61 ({\\scriptsize .55, .68}) &  .71 ({\\scriptsize .65, .78}) &  .81 ({\\scriptsize .76, .87}) &  .53 ({\\scriptsize .46, .61}) &  .62 ({\\scriptsize .55, .69}) \\\\\n",
      "Self-critique    &  .51 ({\\scriptsize .43, .58}) &  .58 ({\\scriptsize .51, .65}) &  .72 ({\\scriptsize .65, .78}) &  .84 ({\\scriptsize .79, .89}) &  .57 ({\\scriptsize .50, .64}) &  .64 ({\\scriptsize .57, .71}) \\\\\n",
      "Zhou-instruction &  .56 ({\\scriptsize .49, .63}) &  .59 ({\\scriptsize .52, .66}) &  .75 ({\\scriptsize .69, .82}) &  .85 ({\\scriptsize .80, .90}) &  .54 ({\\scriptsize .47, .61}) &  .65 ({\\scriptsize .58, .71}) \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1153/4069475834.py:8: FutureWarning: In future versions `DataFrame.to_latex` is expected to utilise the base implementation of `Styler.to_latex` for formatting and rendering. The arguments signature may therefore change. It is recommended instead to use `DataFrame.style.to_latex` which also contains additional functionality.\n",
      "  latex_table = final_df.to_latex(column_format=\"l\" + \"c\" * len(final_df.columns), float_format=\"{:.2f}\".format, escape=False)\n"
     ]
    }
   ],
   "source": [
    "# Combine the DataFrames\n",
    "\n",
    "final_df = mean_pivot.copy()\n",
    "for col in mean_pivot.columns:\n",
    "    final_df[col] = mean_pivot[col] + \" ({\\\\scriptsize \" + ci_lower[col] + \", \" + ci_upper[col] + \"})\"\n",
    "\n",
    "# Apply custom formatting (use LaTeX format for a scientific paper)\n",
    "latex_table = final_df.to_latex(column_format=\"l\" + \"c\" * len(final_df.columns), float_format=\"{:.2f}\".format, escape=False)\n",
    "\n",
    "print(latex_table)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from docx import Document\n",
    "# doc = Document()\n",
    "\n",
    "# df[\"Krippendorff's \" + chr(945)] = df['mean'].round(2).astype(str) + \" (\" + df['ci_lower'].round(2).astype(str) + \", \" + df['ci_upper'].round(2).astype(str) + \")\"\n",
    "\n",
    "# # Drop the original 'mean', 'ci_lower', and 'ci_upper' columns\n",
    "# df = df.drop(columns=['mean', 'ci_lower', 'ci_upper'])\n",
    "\n",
    "# # Add a table to the document\n",
    "# table = doc.add_table(rows=df.shape[0] + 1, cols=df.shape[1])\n",
    "\n",
    "# # Populate the table with the DataFrame content\n",
    "# for i, col in enumerate(df.columns):\n",
    "#     table.cell(0, i).text = col\n",
    "\n",
    "# for i, row in df.iterrows():\n",
    "#     for j, value in enumerate(row):\n",
    "#         table.cell(i + 1, j).text = str(value)\n",
    "\n",
    "# # Save the Word document\n",
    "# name = \"dataframe_\" + \"_\".join(df.columns) + \".docx\"\n",
    "# doc.save(name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### test runs for checking scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 449,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_273/1604803771.py:16: FutureWarning: In a future version of pandas, a length 1 tuple will be returned when iterating over a groupby with a grouper equal to a list of length 1. Don't supply a list with a single grouper to avoid this warning.\n",
      "  for (dataset), group in df.groupby(['dataset']):\n"
     ]
    }
   ],
   "source": [
    "# test run to see if the pooling is correct for the dataset option\n",
    "# bootstrapping per dataset is not needed for the dataset option, we can check if it leads to the same results\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "import scipy.stats\n",
    "\n",
    "n_bootstraps = 1000\n",
    "confidence_level = 0.95\n",
    "np.random.seed(42)  # Set the seed for reproducibility\n",
    "random.seed(42)\n",
    "\n",
    "# Perform bootstrapping and calculate means and confidence intervals\n",
    "results = []\n",
    "\n",
    "for (dataset), group in df.groupby(['dataset']):\n",
    "    unique_datasets = group['model'].unique()\n",
    "    dataset_means = {dataset: [] for dataset in unique_datasets}\n",
    "\n",
    "    for _ in range(n_bootstraps):\n",
    "\n",
    "        random_dataset = random.choice(unique_datasets)\n",
    "        random_group = group[group['model'] == random_dataset]\n",
    "\n",
    "        answer_label = random_group['answer_label'].sample(n=len(random_group), replace=True)\n",
    "        indices = answer_label.index\n",
    "        answer_from_choices = random_group.loc[indices, 'answer_from_choices']\n",
    "\n",
    "        metric = krippendorff.alpha(reliability_data=[list(answer_label), list(answer_from_choices)], level_of_measurement='nominal')\n",
    "        if metric < 0:\n",
    "            metric = 0\n",
    "        dataset_means[random_dataset].append(metric)\n",
    "\n",
    "    all_means = np.concatenate(list(dataset_means.values()))\n",
    "    mean = np.mean(all_means)\n",
    "\n",
    "    # Calculate the pooled standard deviation\n",
    "    dataset_stds = {dataset: np.std(means) for dataset, means in dataset_means.items()}\n",
    "\n",
    "    squared_stds = [std ** 2 for std in dataset_stds.values()]\n",
    "    pooled_std_dev = np.sqrt(sum(squared_stds) / len(squared_stds))\n",
    "\n",
    "    # Calculate the margin of error using the z-score\n",
    "    z_score = scipy.stats.norm.ppf((1 + confidence_level) / 2)\n",
    "    margin_of_error = z_score * (pooled_std_dev / np.sqrt(len(unique_datasets)))\n",
    "\n",
    "    # Compute the confidence interval\n",
    "    ci_lower = mean - margin_of_error\n",
    "    ci_upper = mean + margin_of_error\n",
    "\n",
    "    results.append({\n",
    "        'dataset': dataset,\n",
    "        'mean': mean,\n",
    "        'ci_lower': ci_lower,\n",
    "        'ci_upper': ci_upper\n",
    "    })\n",
    "\n",
    "# Convert the results to a DataFrame\n",
    "results_df = pd.DataFrame(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 450,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>mean</th>\n",
       "      <th>ci_lower</th>\n",
       "      <th>ci_upper</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>commonsense_qa</td>\n",
       "      <td>0.7111</td>\n",
       "      <td>0.6876</td>\n",
       "      <td>0.7346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>med_qa</td>\n",
       "      <td>0.2207</td>\n",
       "      <td>0.1956</td>\n",
       "      <td>0.2459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>medmc_qa</td>\n",
       "      <td>0.3022</td>\n",
       "      <td>0.2737</td>\n",
       "      <td>0.3306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>open_book_qa</td>\n",
       "      <td>0.6512</td>\n",
       "      <td>0.6248</td>\n",
       "      <td>0.6777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>strategy_qa</td>\n",
       "      <td>0.3085</td>\n",
       "      <td>0.2605</td>\n",
       "      <td>0.3565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>worldtree</td>\n",
       "      <td>0.8329</td>\n",
       "      <td>0.8139</td>\n",
       "      <td>0.8519</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          dataset   mean  ci_lower  ci_upper\n",
       "0  commonsense_qa 0.7111    0.6876    0.7346\n",
       "1          med_qa 0.2207    0.1956    0.2459\n",
       "2        medmc_qa 0.3022    0.2737    0.3306\n",
       "3    open_book_qa 0.6512    0.6248    0.6777\n",
       "4     strategy_qa 0.3085    0.2605    0.3565\n",
       "5       worldtree 0.8329    0.8139    0.8519"
      ]
     },
     "execution_count": 450,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test if the pooling is correct for the dataset option\n",
    "results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 451,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>mean</th>\n",
       "      <th>ci_lower</th>\n",
       "      <th>ci_upper</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>commonsense_qa</td>\n",
       "      <td>0.7073</td>\n",
       "      <td>0.6819</td>\n",
       "      <td>0.7314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>med_qa</td>\n",
       "      <td>0.2184</td>\n",
       "      <td>0.1920</td>\n",
       "      <td>0.2456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>medmc_qa</td>\n",
       "      <td>0.3051</td>\n",
       "      <td>0.2749</td>\n",
       "      <td>0.3358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>open_book_qa</td>\n",
       "      <td>0.6516</td>\n",
       "      <td>0.6258</td>\n",
       "      <td>0.6780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>strategy_qa</td>\n",
       "      <td>0.3436</td>\n",
       "      <td>0.2961</td>\n",
       "      <td>0.3935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>worldtree</td>\n",
       "      <td>0.8325</td>\n",
       "      <td>0.8125</td>\n",
       "      <td>0.8525</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          dataset   mean  ci_lower  ci_upper\n",
       "0  commonsense_qa 0.7073    0.6819    0.7314\n",
       "1          med_qa 0.2184    0.1920    0.2456\n",
       "2        medmc_qa 0.3051    0.2749    0.3358\n",
       "3    open_book_qa 0.6516    0.6258    0.6780\n",
       "4     strategy_qa 0.3436    0.2961    0.3935\n",
       "5       worldtree 0.8325    0.8125    0.8525"
      ]
     },
     "execution_count": 451,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# should be very similar to\n",
    "kripp_dataset"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "these two should roughly be the same"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 461,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "model\n",
       "command-xlarge-nightly   0.3103\n",
       "flan-T5-xxl              0.4359\n",
       "gpt-3.5-turbo            0.6212\n",
       "gpt-4                    0.7827\n",
       "text-davinci-002         0.4138\n",
       "text-davinci-003         0.4648\n",
       "dtype: float64"
      ]
     },
     "execution_count": 461,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kripp_dataset_model.pivot_table(values='mean', index='dataset', columns='model').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 462,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>command-xlarge-nightly</td>\n",
       "      <td>0.3135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>flan-T5-xxl</td>\n",
       "      <td>0.4325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gpt-3.5-turbo</td>\n",
       "      <td>0.6334</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gpt-4</td>\n",
       "      <td>0.7835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>0.4185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>text-davinci-003</td>\n",
       "      <td>0.4698</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    model   mean\n",
       "0  command-xlarge-nightly 0.3135\n",
       "1             flan-T5-xxl 0.4325\n",
       "2           gpt-3.5-turbo 0.6334\n",
       "3                   gpt-4 0.7835\n",
       "4        text-davinci-002 0.4185\n",
       "5        text-davinci-003 0.4698"
      ]
     },
     "execution_count": 462,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kripp_model[['model','mean']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # INCORRECT SINCE SUMMING OVER DATASETS\n",
    "# # RESULTS WILL BE TOO HIGH\n",
    "\n",
    "# # KRIPPENDORFF ALPHA all datasets together\n",
    "# # Define bootstrapping parameters\n",
    "# n_bootstraps = 1000\n",
    "# confidence_level = 0.95\n",
    "# np.random.seed(42)  # Set the seed for reproducibility\n",
    "# random.seed(42)\n",
    "\n",
    "# # Perform bootstrapping and calculate means and confidence intervals\n",
    "# results = []\n",
    "\n",
    "# for (model, prompt), group in df.groupby(['model', 'prompt']):\n",
    "#     bootstrapped_means = []\n",
    "#     for _ in range(n_bootstraps):\n",
    "#         answer_label = group['answer_label'].sample(n=len(group), replace=True)\n",
    "#         indices = answer_label.index\n",
    "#         answer_from_choices = group.loc[indices, 'answer_from_choices']\n",
    "\n",
    "#         metric = krippendorff.alpha(reliability_data=[list(answer_label), list(answer_from_choices)],level_of_measurement='nominal')\n",
    "        # if metric < 0:\n",
    "        #     metric = 0\n",
    "#         bootstrapped_means.append(metric)\n",
    "    \n",
    "#     mean = np.mean(bootstrapped_means)\n",
    "#     lower = np.percentile(bootstrapped_means, (1 - confidence_level) / 2 * 100)\n",
    "#     upper = np.percentile(bootstrapped_means, (1 + confidence_level) / 2 * 100)\n",
    "    \n",
    "#     results.append({\n",
    "#         # 'dataset': dataset,\n",
    "#         'model': model,\n",
    "#         'prompt': prompt,\n",
    "#         'mean': mean,\n",
    "#         'ci_lower': lower,\n",
    "#         'ci_upper': upper\n",
    "#     })\n",
    "\n",
    "# # Convert the results to a DataFrame and pivot\n",
    "# results_df = pd.DataFrame(results)\n",
    "\n",
    "# results_df.pivot_table(values='mean', index='model', columns='prompt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Define bootstrapping parameters\n",
    "# n_bootstraps = 1000\n",
    "# confidence_level = 0.95\n",
    "# np.random.seed(42)  # Set the seed for reproducibility\n",
    "# random.seed(42)\n",
    "\n",
    "# # Perform bootstrapping and calculate means and confidence intervals\n",
    "# results = []\n",
    "\n",
    "# for (model, prompt, dataset), group in df.groupby(['model', 'prompt', 'dataset']):\n",
    "#     bootstrapped_means = []\n",
    "#     for _ in range(n_bootstraps):\n",
    "#         resampled = group['correct_answer'].sample(n=len(group), replace=True)\n",
    "#         bootstrapped_means.append(resampled.mean())\n",
    "    \n",
    "#     mean = np.mean(bootstrapped_means)\n",
    "#     lower = np.percentile(bootstrapped_means, (1 - confidence_level) / 2 * 100)\n",
    "#     upper = np.percentile(bootstrapped_means, (1 + confidence_level) / 2 * 100)\n",
    "    \n",
    "#     results.append({\n",
    "#         'dataset': dataset,\n",
    "#         'model': model,\n",
    "#         'prompt': prompt,\n",
    "#         'mean': mean,\n",
    "#         'ci_lower': lower,\n",
    "#         'ci_upper': upper\n",
    "#     })\n",
    "\n",
    "# # Convert the results to a DataFrame and pivot\n",
    "# results_df = pd.DataFrame(results)\n",
    "# mean_pivot = results_df.pivot_table(values='mean', index='model', columns='prompt')\n",
    "# ci_pivot_lower = results_df.pivot_table(values='ci_lower', index='model', columns='prompt')\n",
    "# ci_pivot_upper = results_df.pivot_table(values='ci_upper', index='model', columns='prompt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # ACCURACY BOOTSRTAPPING\n",
    "# # Define bootstrapping parameters\n",
    "# n_bootstraps = 1000\n",
    "# confidence_level = 0.95\n",
    "# np.random.seed(42)  # Set the seed for reproducibility\n",
    "# random.seed(42)\n",
    "\n",
    "# # Perform bootstrapping and calculate means and confidence intervals\n",
    "# results = []\n",
    "\n",
    "# for (model, prompt), group in df.groupby(['model', 'prompt']):\n",
    "#     bootstrapped_means = []\n",
    "#     for _ in range(n_bootstraps):\n",
    "#         resampled = group['correct_answer'].sample(n=len(group), replace=True)\n",
    "#         bootstrapped_means.append(resampled.mean())\n",
    "    \n",
    "#     mean = np.mean(bootstrapped_means)\n",
    "#     lower = np.percentile(bootstrapped_means, (1 - confidence_level) / 2 * 100)\n",
    "#     upper = np.percentile(bootstrapped_means, (1 + confidence_level) / 2 * 100)\n",
    "    \n",
    "#     results.append({\n",
    "#         'model': model,\n",
    "#         'prompt': prompt,\n",
    "#         'mean': mean,\n",
    "#         'ci_lower': lower,\n",
    "#         'ci_upper': upper\n",
    "#     })\n",
    "\n",
    "# # Convert the results to a DataFrame and pivot\n",
    "# results_df = pd.DataFrame(results)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### inspection of the dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 481,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_df = df[(df.dataset == \"strategy_qa\")& (df.model == \"command-xlarge-nightly\") & (df.prompt == \"None_zhou-01\")]\n",
    "# filtered_df.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filtered_df[['answer', 'answer_label']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filtered_df[\"correct_answer\"].unique(), filtered_df[\"answer_from_choices\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filtered_df[\"correct_answer\"].value_counts(), filtered_df[\"answer_from_choices\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filtered_df[\"correct_answer\"].count(), filtered_df[\"answer_from_choices\"].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0493"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "krippendorff.alpha(reliability_data=[list(filtered_df[\"answer_label\"]), list(filtered_df[\"answer_from_choices\"].replace(\"\", np.nan))],level_of_measurement='nominal')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filtered_df[[\"answer\",\"answer_from_choices\"]][:200]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df.groupby(['dataset'])['number_choices'].unique()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### functions extra, non pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def table_function_new(eval:dict):\n",
    "    import pandas as pd\n",
    "    eval_dict = pd.json_normalize(eval).to_dict('records')[0]\n",
    "    eval_list = list(eval_dict.keys())\n",
    "    datasets = sorted(list(eval.keys()))\n",
    "\n",
    "    models = []\n",
    "    prompts = []\n",
    "    for i in eval_list:\n",
    "        # fast fix for chat gpt model:\n",
    "        i = i.replace(\"gpt-3.5-turbo\",\"gpt-3-5-turbo\")\n",
    "        dataset,split,metric,model,prompt = i.split(\".\")\n",
    "        model = model.replace(\"gpt-3-5-turbo\",\"gpt-3.5-turbo\")\n",
    "        if model not in models:\n",
    "            models.append(model)\n",
    "        if prompt not in prompts:\n",
    "            prompts.append(prompt)\n",
    "            \n",
    "    models = sorted(models)\n",
    "\n",
    "    if \"None_None_None\" in prompts: prompts.remove(\"None_None_None\")\n",
    "\n",
    "    # no instructions implemented yet\n",
    "    # instructions = []\n",
    "    cot_triggers = []\n",
    "    for i in prompts:\n",
    "        instruction, cot_trigger, _ = i.split(\"_\")\n",
    "        # if instruction not in instructions:\n",
    "        #     instructions.append(instruction)\n",
    "\n",
    "        # old: only cot_trigger\n",
    "        # if cot_trigger not in cot_triggers:\n",
    "        #     cot_triggers.append(cot_trigger)\n",
    "\n",
    "        # new: instruction + cot_trigger\n",
    "        if cot_trigger not in cot_triggers:\n",
    "            cot_triggers.append(instruction + \"_\" + cot_trigger)\n",
    "\n",
    "    cot_triggers = sorted(cot_triggers)\n",
    "\n",
    "    cot_trigger_header = sorted(cot_triggers*len(models))\n",
    "    model_header = models*len(cot_triggers)\n",
    "\n",
    "    # Create a dictionary to store the data\n",
    "    data_dict = {}\n",
    "    for k,v in eval_dict.items():\n",
    "        # fast fix for chat gpt model:\n",
    "        k = k.replace(\"gpt-3.5-turbo\",\"gpt-3-5-turbo\")\n",
    "        dataset,split,metric,model,prompt = k.split(\".\")\n",
    "        model = model.replace(\"gpt-3-5-turbo\",\"gpt-3.5-turbo\")\n",
    "        instruction, cot_trigger, _ = prompt.split(\"_\")\n",
    "        # old: only cot_trigger\n",
    "        # df.loc[dataset, (cot_trigger, model)] = v\n",
    "\n",
    "        # new: instruction + cot_trigger\n",
    "        col_name = instruction + \"_\" + cot_trigger + \"_\" + model\n",
    "        if dataset not in data_dict:\n",
    "            data_dict[dataset] = {}\n",
    "        data_dict[dataset][col_name] = v\n",
    "\n",
    "    # Create a dataframe from the dictionary\n",
    "    df = pd.DataFrame.from_dict(data_dict, orient='index')\n",
    "\n",
    "    # Calculate the average row and add it to the dataframe\n",
    "    df.loc['Average'] = df.mean()\n",
    "\n",
    "    # Round the dataframe to 2 decimal places\n",
    "    df = df.round(2)\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>None_None_command-xlarge-nightly</th>\n",
       "      <th>None_kojima-01_command-xlarge-nightly</th>\n",
       "      <th>None_zhou-01_command-xlarge-nightly</th>\n",
       "      <th>qa-10_None_command-xlarge-nightly</th>\n",
       "      <th>qa-12_None_command-xlarge-nightly</th>\n",
       "      <th>qa-13_None_command-xlarge-nightly</th>\n",
       "      <th>qa-16_None_command-xlarge-nightly</th>\n",
       "      <th>qa-17_None_command-xlarge-nightly</th>\n",
       "      <th>refl-01_None_command-xlarge-nightly</th>\n",
       "      <th>zhou-01-ins_None_command-xlarge-nightly</th>\n",
       "      <th>...</th>\n",
       "      <th>None_None_text-davinci-003</th>\n",
       "      <th>None_kojima-01_text-davinci-003</th>\n",
       "      <th>None_zhou-01_text-davinci-003</th>\n",
       "      <th>qa-10_None_text-davinci-003</th>\n",
       "      <th>qa-12_None_text-davinci-003</th>\n",
       "      <th>qa-13_None_text-davinci-003</th>\n",
       "      <th>qa-16_None_text-davinci-003</th>\n",
       "      <th>qa-17_None_text-davinci-003</th>\n",
       "      <th>refl-01_None_text-davinci-003</th>\n",
       "      <th>zhou-01-ins_None_text-davinci-003</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>commonsense_qa</th>\n",
       "      <td>0.52</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.73</td>\n",
       "      <td>...</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.79</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>med_qa</th>\n",
       "      <td>0.18</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.33</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.21</td>\n",
       "      <td>0.24</td>\n",
       "      <td>...</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.33</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.33</td>\n",
       "      <td>0.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>medmc_qa</th>\n",
       "      <td>0.27</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.33</td>\n",
       "      <td>...</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>open_book_qa</th>\n",
       "      <td>0.58</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.55</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.64</td>\n",
       "      <td>...</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.79</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.79</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>strategy_qa</th>\n",
       "      <td>0.55</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.33</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.09</td>\n",
       "      <td>...</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.55</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>worldtree</th>\n",
       "      <td>0.61</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.82</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.79</td>\n",
       "      <td>0.82</td>\n",
       "      <td>...</td>\n",
       "      <td>0.94</td>\n",
       "      <td>0.91</td>\n",
       "      <td>0.82</td>\n",
       "      <td>0.82</td>\n",
       "      <td>0.91</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.82</td>\n",
       "      <td>0.91</td>\n",
       "      <td>0.91</td>\n",
       "      <td>0.88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Average</th>\n",
       "      <td>0.45</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.47</td>\n",
       "      <td>0.43</td>\n",
       "      <td>0.47</td>\n",
       "      <td>...</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.56</td>\n",
       "      <td>0.59</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.63</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7 rows × 60 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                None_None_command-xlarge-nightly  \\\n",
       "commonsense_qa                              0.52   \n",
       "med_qa                                      0.18   \n",
       "medmc_qa                                    0.27   \n",
       "open_book_qa                                0.58   \n",
       "strategy_qa                                 0.55   \n",
       "worldtree                                   0.61   \n",
       "Average                                     0.45   \n",
       "\n",
       "                None_kojima-01_command-xlarge-nightly  \\\n",
       "commonsense_qa                                   0.48   \n",
       "med_qa                                           0.30   \n",
       "medmc_qa                                         0.15   \n",
       "open_book_qa                                     0.42   \n",
       "strategy_qa                                      0.58   \n",
       "worldtree                                        0.61   \n",
       "Average                                          0.42   \n",
       "\n",
       "                None_zhou-01_command-xlarge-nightly  \\\n",
       "commonsense_qa                                 0.73   \n",
       "med_qa                                         0.27   \n",
       "medmc_qa                                       0.42   \n",
       "open_book_qa                                   0.61   \n",
       "strategy_qa                                    0.24   \n",
       "worldtree                                      0.76   \n",
       "Average                                        0.51   \n",
       "\n",
       "                qa-10_None_command-xlarge-nightly  \\\n",
       "commonsense_qa                               0.70   \n",
       "med_qa                                       0.27   \n",
       "medmc_qa                                     0.27   \n",
       "open_book_qa                                 0.58   \n",
       "strategy_qa                                  0.12   \n",
       "worldtree                                    0.73   \n",
       "Average                                      0.44   \n",
       "\n",
       "                qa-12_None_command-xlarge-nightly  \\\n",
       "commonsense_qa                               0.70   \n",
       "med_qa                                       0.27   \n",
       "medmc_qa                                     0.30   \n",
       "open_book_qa                                 0.58   \n",
       "strategy_qa                                  0.09   \n",
       "worldtree                                    0.82   \n",
       "Average                                      0.46   \n",
       "\n",
       "                qa-13_None_command-xlarge-nightly  \\\n",
       "commonsense_qa                               0.76   \n",
       "med_qa                                       0.24   \n",
       "medmc_qa                                     0.36   \n",
       "open_book_qa                                 0.58   \n",
       "strategy_qa                                  0.12   \n",
       "worldtree                                    0.85   \n",
       "Average                                      0.48   \n",
       "\n",
       "                qa-16_None_command-xlarge-nightly  \\\n",
       "commonsense_qa                               0.67   \n",
       "med_qa                                       0.33   \n",
       "medmc_qa                                     0.36   \n",
       "open_book_qa                                 0.55   \n",
       "strategy_qa                                  0.33   \n",
       "worldtree                                    0.85   \n",
       "Average                                      0.52   \n",
       "\n",
       "                qa-17_None_command-xlarge-nightly  \\\n",
       "commonsense_qa                               0.61   \n",
       "med_qa                                       0.30   \n",
       "medmc_qa                                     0.36   \n",
       "open_book_qa                                 0.61   \n",
       "strategy_qa                                  0.30   \n",
       "worldtree                                    0.64   \n",
       "Average                                      0.47   \n",
       "\n",
       "                refl-01_None_command-xlarge-nightly  \\\n",
       "commonsense_qa                                 0.70   \n",
       "med_qa                                         0.21   \n",
       "medmc_qa                                       0.24   \n",
       "open_book_qa                                   0.61   \n",
       "strategy_qa                                    0.06   \n",
       "worldtree                                      0.79   \n",
       "Average                                        0.43   \n",
       "\n",
       "                zhou-01-ins_None_command-xlarge-nightly  ...  \\\n",
       "commonsense_qa                                     0.73  ...   \n",
       "med_qa                                             0.24  ...   \n",
       "medmc_qa                                           0.33  ...   \n",
       "open_book_qa                                       0.64  ...   \n",
       "strategy_qa                                        0.09  ...   \n",
       "worldtree                                          0.82  ...   \n",
       "Average                                            0.47  ...   \n",
       "\n",
       "                None_None_text-davinci-003  None_kojima-01_text-davinci-003  \\\n",
       "commonsense_qa                        0.73                             0.67   \n",
       "med_qa                                0.30                             0.33   \n",
       "medmc_qa                              0.36                             0.36   \n",
       "open_book_qa                          0.64                             0.58   \n",
       "strategy_qa                           0.61                             0.64   \n",
       "worldtree                             0.94                             0.91   \n",
       "Average                               0.60                             0.58   \n",
       "\n",
       "                None_zhou-01_text-davinci-003  qa-10_None_text-davinci-003  \\\n",
       "commonsense_qa                           0.67                         0.70   \n",
       "med_qa                                   0.39                         0.36   \n",
       "medmc_qa                                 0.48                         0.45   \n",
       "open_book_qa                             0.79                         0.73   \n",
       "strategy_qa                              0.64                         0.58   \n",
       "worldtree                                0.82                         0.82   \n",
       "Average                                  0.63                         0.61   \n",
       "\n",
       "                qa-12_None_text-davinci-003  qa-13_None_text-davinci-003  \\\n",
       "commonsense_qa                         0.79                         0.67   \n",
       "med_qa                                 0.30                         0.39   \n",
       "medmc_qa                               0.36                         0.36   \n",
       "open_book_qa                           0.79                         0.61   \n",
       "strategy_qa                            0.67                         0.70   \n",
       "worldtree                              0.91                         0.85   \n",
       "Average                                0.64                         0.60   \n",
       "\n",
       "                qa-16_None_text-davinci-003  qa-17_None_text-davinci-003  \\\n",
       "commonsense_qa                         0.58                         0.64   \n",
       "med_qa                                 0.36                         0.24   \n",
       "medmc_qa                               0.39                         0.42   \n",
       "open_book_qa                           0.64                         0.73   \n",
       "strategy_qa                            0.55                         0.58   \n",
       "worldtree                              0.82                         0.91   \n",
       "Average                                0.56                         0.59   \n",
       "\n",
       "                refl-01_None_text-davinci-003  \\\n",
       "commonsense_qa                           0.76   \n",
       "med_qa                                   0.33   \n",
       "medmc_qa                                 0.39   \n",
       "open_book_qa                             0.64   \n",
       "strategy_qa                              0.64   \n",
       "worldtree                                0.91   \n",
       "Average                                  0.61   \n",
       "\n",
       "                zhou-01-ins_None_text-davinci-003  \n",
       "commonsense_qa                               0.73  \n",
       "med_qa                                       0.36  \n",
       "medmc_qa                                     0.33  \n",
       "open_book_qa                                 0.82  \n",
       "strategy_qa                                  0.64  \n",
       "worldtree                                    0.88  \n",
       "Average                                      0.63  \n",
       "\n",
       "[7 rows x 60 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = table_function_new(eval)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>None_None_command-xlarge-nightly</th>\n",
       "      <th>None_kojima-01_command-xlarge-nightly</th>\n",
       "      <th>None_zhou-01_command-xlarge-nightly</th>\n",
       "      <th>qa-10_None_command-xlarge-nightly</th>\n",
       "      <th>qa-12_None_command-xlarge-nightly</th>\n",
       "      <th>qa-13_None_command-xlarge-nightly</th>\n",
       "      <th>qa-16_None_command-xlarge-nightly</th>\n",
       "      <th>qa-17_None_command-xlarge-nightly</th>\n",
       "      <th>refl-01_None_command-xlarge-nightly</th>\n",
       "      <th>zhou-01-ins_None_command-xlarge-nightly</th>\n",
       "      <th>...</th>\n",
       "      <th>None_None_text-davinci-003</th>\n",
       "      <th>None_kojima-01_text-davinci-003</th>\n",
       "      <th>None_zhou-01_text-davinci-003</th>\n",
       "      <th>qa-10_None_text-davinci-003</th>\n",
       "      <th>qa-12_None_text-davinci-003</th>\n",
       "      <th>qa-13_None_text-davinci-003</th>\n",
       "      <th>qa-16_None_text-davinci-003</th>\n",
       "      <th>qa-17_None_text-davinci-003</th>\n",
       "      <th>refl-01_None_text-davinci-003</th>\n",
       "      <th>zhou-01-ins_None_text-davinci-003</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Average</th>\n",
       "      <td>0.45</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.47</td>\n",
       "      <td>0.43</td>\n",
       "      <td>0.47</td>\n",
       "      <td>...</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.56</td>\n",
       "      <td>0.59</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.63</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 60 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         None_None_command-xlarge-nightly  \\\n",
       "Average                              0.45   \n",
       "\n",
       "         None_kojima-01_command-xlarge-nightly  \\\n",
       "Average                                   0.42   \n",
       "\n",
       "         None_zhou-01_command-xlarge-nightly  \\\n",
       "Average                                 0.51   \n",
       "\n",
       "         qa-10_None_command-xlarge-nightly  qa-12_None_command-xlarge-nightly  \\\n",
       "Average                               0.44                               0.46   \n",
       "\n",
       "         qa-13_None_command-xlarge-nightly  qa-16_None_command-xlarge-nightly  \\\n",
       "Average                               0.48                               0.52   \n",
       "\n",
       "         qa-17_None_command-xlarge-nightly  \\\n",
       "Average                               0.47   \n",
       "\n",
       "         refl-01_None_command-xlarge-nightly  \\\n",
       "Average                                 0.43   \n",
       "\n",
       "         zhou-01-ins_None_command-xlarge-nightly  ...  \\\n",
       "Average                                     0.47  ...   \n",
       "\n",
       "         None_None_text-davinci-003  None_kojima-01_text-davinci-003  \\\n",
       "Average                         0.6                             0.58   \n",
       "\n",
       "         None_zhou-01_text-davinci-003  qa-10_None_text-davinci-003  \\\n",
       "Average                           0.63                         0.61   \n",
       "\n",
       "         qa-12_None_text-davinci-003  qa-13_None_text-davinci-003  \\\n",
       "Average                         0.64                          0.6   \n",
       "\n",
       "         qa-16_None_text-davinci-003  qa-17_None_text-davinci-003  \\\n",
       "Average                         0.56                         0.59   \n",
       "\n",
       "         refl-01_None_text-davinci-003  zhou-01-ins_None_text-davinci-003  \n",
       "Average                           0.61                               0.63  \n",
       "\n",
       "[1 rows x 60 columns]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = (df[df.index==\"Average\"])\n",
    "#df.to_csv(\"Average_values_df.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json, re\n",
    "import pandas as pd\n",
    "def clean_column(df, col_name):\n",
    "    pattern = r\"^.?'(.?)'.?'(.?)'.?'(.?)'.?'(.?)'.*$\"\n",
    "    return(df[col_name].apply(lambda x: re.sub(pattern, r\"'\\3\", x)[1:]))\n",
    "\n",
    "def json_to_dataframe_3(json_data):\n",
    "    df_data = []\n",
    "    for category, data in json_data.items():\n",
    "        for subset, questions in data.items():\n",
    "            for question in questions:\n",
    "                for generated_cot in question['generated_cot']:\n",
    "                    row = {\n",
    "                        'dataset': category,\n",
    "                        'split': subset,\n",
    "                        'id': question['id'],\n",
    "                        'model': generated_cot['model'],\n",
    "                        'instruction': generated_cot['instruction'],\n",
    "                        'cot_trigger': generated_cot[\"cot_trigger\"],\n",
    "                        'generated_cot': generated_cot['cot'],\n",
    "                        'correct_answer': generated_cot['answers'][0]['correct_answer'],\n",
    "                    }\n",
    "                    df_data.append(row)\n",
    "    df = pd.DataFrame(df_data)\n",
    "    # df[\"model\"] = clean_column(df, \"model\")\n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"'name': 'command-xlarge-nightly', 'temperature': 0, 'max_tokens': 512}\""
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"model\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>split</th>\n",
       "      <th>id</th>\n",
       "      <th>model</th>\n",
       "      <th>instruction</th>\n",
       "      <th>cot_trigger</th>\n",
       "      <th>generated_cot</th>\n",
       "      <th>correct_answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>commonsense_qa</td>\n",
       "      <td>validation</td>\n",
       "      <td>9aff72f0c480c2b4edde45bd2e7e4870</td>\n",
       "      <td>'name': 'command-xlarge-nightly', 'temperature...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>The main purpose of farmers is to supply food.</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>commonsense_qa</td>\n",
       "      <td>validation</td>\n",
       "      <td>9aff72f0c480c2b4edde45bd2e7e4870</td>\n",
       "      <td>'name': 'text-davinci-003', 'temperature': 0, ...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>E) supply food</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>commonsense_qa</td>\n",
       "      <td>validation</td>\n",
       "      <td>9aff72f0c480c2b4edde45bd2e7e4870</td>\n",
       "      <td>'name': 'flan-T5-xxl', 'temperature': 0, 'max_...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>E</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>commonsense_qa</td>\n",
       "      <td>validation</td>\n",
       "      <td>9aff72f0c480c2b4edde45bd2e7e4870</td>\n",
       "      <td>'name': 'gpt-3.5-turbo', 'temperature': 0, 'ma...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>E) supply food</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>commonsense_qa</td>\n",
       "      <td>validation</td>\n",
       "      <td>9aff72f0c480c2b4edde45bd2e7e4870</td>\n",
       "      <td>'name': 'text-davinci-002', 'temperature': 0, ...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>\\nE) supply food</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11875</th>\n",
       "      <td>worldtree</td>\n",
       "      <td>test</td>\n",
       "      <td>worldtree_test_1659</td>\n",
       "      <td>'name': 'flan-T5-xxl', 'temperature': 0, 'max_...</td>\n",
       "      <td>None</td>\n",
       "      <td>zhou-01</td>\n",
       "      <td>C is the dependent variable. The dependent var...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11876</th>\n",
       "      <td>worldtree</td>\n",
       "      <td>test</td>\n",
       "      <td>worldtree_test_1659</td>\n",
       "      <td>'name': 'text-davinci-003', 'temperature': 0, ...</td>\n",
       "      <td>None</td>\n",
       "      <td>zhou-01</td>\n",
       "      <td>\\n\\nStep 1: The vertical axis on a graph is t...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11877</th>\n",
       "      <td>worldtree</td>\n",
       "      <td>test</td>\n",
       "      <td>worldtree_test_1659</td>\n",
       "      <td>'name': 'text-davinci-002', 'temperature': 0, ...</td>\n",
       "      <td>None</td>\n",
       "      <td>zhou-01</td>\n",
       "      <td>\\n\\nThe first thing we need to do is identify...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11878</th>\n",
       "      <td>worldtree</td>\n",
       "      <td>test</td>\n",
       "      <td>worldtree_test_1659</td>\n",
       "      <td>'name': 'command-xlarge-nightly', 'temperature...</td>\n",
       "      <td>None</td>\n",
       "      <td>zhou-01</td>\n",
       "      <td>\\nThe y-axis is the vertical axis on a graph. ...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11879</th>\n",
       "      <td>worldtree</td>\n",
       "      <td>test</td>\n",
       "      <td>worldtree_test_1659</td>\n",
       "      <td>'name': 'gpt-4', 'temperature': 0, 'max_tokens...</td>\n",
       "      <td>None</td>\n",
       "      <td>kojima-01</td>\n",
       "      <td>The vertical axis on a graph is the y-axis, wh...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11880 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              dataset       split                                id  \\\n",
       "0      commonsense_qa  validation  9aff72f0c480c2b4edde45bd2e7e4870   \n",
       "1      commonsense_qa  validation  9aff72f0c480c2b4edde45bd2e7e4870   \n",
       "2      commonsense_qa  validation  9aff72f0c480c2b4edde45bd2e7e4870   \n",
       "3      commonsense_qa  validation  9aff72f0c480c2b4edde45bd2e7e4870   \n",
       "4      commonsense_qa  validation  9aff72f0c480c2b4edde45bd2e7e4870   \n",
       "...               ...         ...                               ...   \n",
       "11875       worldtree        test               worldtree_test_1659   \n",
       "11876       worldtree        test               worldtree_test_1659   \n",
       "11877       worldtree        test               worldtree_test_1659   \n",
       "11878       worldtree        test               worldtree_test_1659   \n",
       "11879       worldtree        test               worldtree_test_1659   \n",
       "\n",
       "                                                   model instruction  \\\n",
       "0      'name': 'command-xlarge-nightly', 'temperature...        None   \n",
       "1      'name': 'text-davinci-003', 'temperature': 0, ...        None   \n",
       "2      'name': 'flan-T5-xxl', 'temperature': 0, 'max_...        None   \n",
       "3      'name': 'gpt-3.5-turbo', 'temperature': 0, 'ma...        None   \n",
       "4      'name': 'text-davinci-002', 'temperature': 0, ...        None   \n",
       "...                                                  ...         ...   \n",
       "11875  'name': 'flan-T5-xxl', 'temperature': 0, 'max_...        None   \n",
       "11876  'name': 'text-davinci-003', 'temperature': 0, ...        None   \n",
       "11877  'name': 'text-davinci-002', 'temperature': 0, ...        None   \n",
       "11878  'name': 'command-xlarge-nightly', 'temperature...        None   \n",
       "11879  'name': 'gpt-4', 'temperature': 0, 'max_tokens...        None   \n",
       "\n",
       "      cot_trigger                                      generated_cot  \\\n",
       "0            None     The main purpose of farmers is to supply food.   \n",
       "1            None                                     E) supply food   \n",
       "2            None                                                  E   \n",
       "3            None                                     E) supply food   \n",
       "4            None                                   \\nE) supply food   \n",
       "...           ...                                                ...   \n",
       "11875     zhou-01  C is the dependent variable. The dependent var...   \n",
       "11876     zhou-01   \\n\\nStep 1: The vertical axis on a graph is t...   \n",
       "11877     zhou-01   \\n\\nThe first thing we need to do is identify...   \n",
       "11878     zhou-01  \\nThe y-axis is the vertical axis on a graph. ...   \n",
       "11879   kojima-01  The vertical axis on a graph is the y-axis, wh...   \n",
       "\n",
       "       correct_answer  \n",
       "0                True  \n",
       "1                True  \n",
       "2                True  \n",
       "3                True  \n",
       "4                True  \n",
       "...               ...  \n",
       "11875            True  \n",
       "11876           False  \n",
       "11877            True  \n",
       "11878            True  \n",
       "11879            True  \n",
       "\n",
       "[11880 rows x 8 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = \"/home/kon/work/ThoughtSource/libs/cot/cot/datasets/thoughtsource/thoughtsource_33_paper.json\"\n",
    "\n",
    "\n",
    "with open(path, \"r\") as read_file:\n",
    "    data = json.load(read_file)\n",
    "\n",
    "df = json_to_dataframe_3(data)\n",
    "\n",
    "df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
